{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoding=utf-8\n",
    "import os.path as osp\n",
    "import os\n",
    "import copy\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "from torch.nn import Linear\n",
    "from sklearn.metrics import average_precision_score, roc_auc_score\n",
    "from torch_geometric.data import TemporalData\n",
    "from torch_geometric.datasets import JODIEDataset\n",
    "from torch_geometric.datasets import ICEWS18\n",
    "from torch_geometric.nn import TGNMemory, TransformerConv\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch_geometric.nn.models.tgn import (LastNeighborLoader, IdentityMessage, MeanAggregator,\n",
    "                                           LastAggregator)\n",
    "from torch_geometric import *\n",
    "from torch_geometric.utils import negative_sampling\n",
    "from tqdm import tqdm\n",
    "import networkx as nx\n",
    "import numpy as np\n",
    "import math\n",
    "import copy\n",
    "import re\n",
    "import time\n",
    "import json\n",
    "import pandas as pd\n",
    "from random import choice\n",
    "import gc\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "# device = 'cpu'\n",
    "# msg structure:    [src_node_feature,edge_attr,dst_node_feature]\n",
    "\n",
    "# compute the best partition \n",
    "import datetime\n",
    "# import community as community_louvain\n",
    "\n",
    "import xxhash\n",
    "\n",
    "# Find the edge index which the edge vector is corresponding to\n",
    "def tensor_find(t,x):\n",
    "    t_np=t.cpu().numpy()\n",
    "    idx=np.argwhere(t_np==x)\n",
    "    return idx[0][0]+1\n",
    "\n",
    "\n",
    "def std(t):\n",
    "    t = np.array(t)\n",
    "    return np.std(t)\n",
    "\n",
    "\n",
    "def var(t):\n",
    "    t = np.array(t)\n",
    "    return np.var(t)\n",
    "\n",
    "\n",
    "def mean(t):\n",
    "    t = np.array(t)\n",
    "    return np.mean(t)\n",
    "\n",
    "def hashgen(l):\n",
    "    \"\"\"Generate a single hash value from a list. @l is a list of\n",
    "    string values, which can be properties of a node/edge. This\n",
    "    function returns a single hashed integer value.\"\"\"\n",
    "    hasher = xxhash.xxh64()\n",
    "    for e in l:\n",
    "        hasher.update(e)\n",
    "    return hasher.intdigest()\n",
    "\n",
    "\n",
    "def cal_pos_edges_loss(link_pred_ratio):\n",
    "    loss=[]\n",
    "    for i in link_pred_ratio:\n",
    "        loss.append(criterion(i,torch.ones(1)))\n",
    "    return torch.tensor(loss)\n",
    "\n",
    "def cal_pos_edges_loss_multiclass(link_pred_ratio,labels):\n",
    "    loss=[] \n",
    "    for i in range(len(link_pred_ratio)):\n",
    "        loss.append(criterion(link_pred_ratio[i].reshape(1,-1),labels[i].reshape(-1)))\n",
    "    return torch.tensor(loss)\n",
    "\n",
    "def cal_pos_edges_loss_autoencoder(decoded,msg):\n",
    "    loss=[] \n",
    "    for i in range(len(decoded)):\n",
    "        loss.append(criterion(decoded[i].reshape(1,-1),msg[i].reshape(-1)))\n",
    "    return torch.tensor(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.set_autosave_interval(120000)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Autosaving every 120 seconds\n"
     ]
    }
   ],
   "source": [
    "%autosave 120  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime, timezone\n",
    "import time\n",
    "import pytz\n",
    "from time import mktime\n",
    "from datetime import datetime\n",
    "import time\n",
    "def ns_time_to_datetime(ns):\n",
    "    \"\"\"\n",
    "    :param ns: int nano timestamp\n",
    "    :return: datetime   format: 2013-10-10 23:40:00.000000000\n",
    "    \"\"\"\n",
    "    dt = datetime.fromtimestamp(int(ns) // 1000000000)\n",
    "    s = dt.strftime('%Y-%m-%d %H:%M:%S')\n",
    "    s += '.' + str(int(int(ns) % 1000000000)).zfill(9)\n",
    "    return s\n",
    "\n",
    "def ns_time_to_datetime_US(ns):\n",
    "    \"\"\"\n",
    "    :param ns: int nano timestamp\n",
    "    :return: datetime   format: 2013-10-10 23:40:00.000000000\n",
    "    \"\"\"\n",
    "    tz = pytz.timezone('US/Eastern')\n",
    "    dt = pytz.datetime.datetime.fromtimestamp(int(ns) // 1000000000, tz)\n",
    "    s = dt.strftime('%Y-%m-%d %H:%M:%S')\n",
    "    s += '.' + str(int(int(ns) % 1000000000)).zfill(9)\n",
    "    return s\n",
    "\n",
    "def time_to_datetime_US(s):\n",
    "    \"\"\"\n",
    "    :param ns: int nano timestamp\n",
    "    :return: datetime   format: 2013-10-10 23:40:00\n",
    "    \"\"\"\n",
    "    tz = pytz.timezone('US/Eastern')\n",
    "    dt = pytz.datetime.datetime.fromtimestamp(int(s), tz)\n",
    "    s = dt.strftime('%Y-%m-%d %H:%M:%S')\n",
    "\n",
    "    return s\n",
    "\n",
    "def datetime_to_ns_time(date):\n",
    "    \"\"\"\n",
    "    :param date: str   format: %Y-%m-%d %H:%M:%S   e.g. 2013-10-10 23:40:00\n",
    "    :return: nano timestamp\n",
    "    \"\"\"\n",
    "    timeArray = time.strptime(date, \"%Y-%m-%d %H:%M:%S\")\n",
    "    timeStamp = int(time.mktime(timeArray))\n",
    "    timeStamp = timeStamp * 1000000000\n",
    "    return timeStamp\n",
    "\n",
    "def datetime_to_ns_time_US(date):\n",
    "    \"\"\"\n",
    "    :param date: str   format: %Y-%m-%d %H:%M:%S   e.g. 2013-10-10 23:40:00\n",
    "    :return: nano timestamp\n",
    "    \"\"\"\n",
    "    tz = pytz.timezone('US/Eastern')\n",
    "    timeArray = time.strptime(date, \"%Y-%m-%d %H:%M:%S\")\n",
    "    dt = datetime.fromtimestamp(mktime(timeArray))\n",
    "    timestamp = tz.localize(dt)\n",
    "    timestamp = timestamp.timestamp()\n",
    "    timeStamp = timestamp * 1000000000\n",
    "    return int(timeStamp)\n",
    "\n",
    "def datetime_to_timestamp_US(date):\n",
    "    \"\"\"\n",
    "    :param date: str   format: %Y-%m-%d %H:%M:%S   e.g. 2013-10-10 23:40:00\n",
    "    :return: nano timestamp\n",
    "    \"\"\"\n",
    "    tz = pytz.timezone('US/Eastern')\n",
    "    timeArray = time.strptime(date, \"%Y-%m-%d %H:%M:%S\")\n",
    "    dt = datetime.fromtimestamp(mktime(timeArray))\n",
    "    timestamp = tz.localize(dt)\n",
    "    timestamp = timestamp.timestamp()\n",
    "    timeStamp = timestamp\n",
    "    return int(timeStamp)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import psycopg2\n",
    "\n",
    "from psycopg2 import extras as ex\n",
    "connect = psycopg2.connect(database = 'tc_e5_cadets_dataset_db',\n",
    "                           host = '/var/run/postgresql/',\n",
    "                           user = 'postgres',\n",
    "                           password = 'postgres',\n",
    "                           port = '5432'\n",
    "                          )\n",
    "\n",
    "\n",
    "cur = connect.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_5_8=torch.load(\"./train_graph/graph_5_8.TemporalData.simple\").to(device=device)\n",
    "graph_5_9=torch.load(\"./train_graph/graph_5_9.TemporalData.simple\").to(device=device)\n",
    "graph_5_11=torch.load(\"./train_graph/graph_5_11.TemporalData.simple\").to(device=device)\n",
    "\n",
    "\n",
    "train_data=graph_5_8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Constructing the map for nodeid to msg\n",
    "sql=\"select * from node2id ORDER BY index_id;\"\n",
    "cur.execute(sql)\n",
    "rows = cur.fetchall()\n",
    "\n",
    "nodeid2msg={}  # nodeid => msg and node hash => nodeid\n",
    "for i in rows:\n",
    "    nodeid2msg[i[0]]=i[-1]\n",
    "    nodeid2msg[i[-1]]={i[1]:i[2]}  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "rel2id={1: 'EVENT_CLOSE',\n",
    " 'EVENT_CLOSE': 1,\n",
    " 2: 'EVENT_OPEN',\n",
    " 'EVENT_OPEN': 2,\n",
    " 3: 'EVENT_READ',\n",
    " 'EVENT_READ': 3,\n",
    " 4: 'EVENT_WRITE',\n",
    " 'EVENT_WRITE': 4,\n",
    " 5: 'EVENT_EXECUTE',\n",
    " 'EVENT_EXECUTE': 5,\n",
    " 6: 'EVENT_RECVFROM',\n",
    " 'EVENT_RECVFROM': 6,\n",
    " 7: 'EVENT_RECVMSG',\n",
    " 'EVENT_RECVMSG': 7,\n",
    " 8: 'EVENT_SENDMSG',\n",
    " 'EVENT_SENDMSG': 8,\n",
    " 9: 'EVENT_SENDTO',\n",
    " 'EVENT_SENDTO': 9}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_data, val_data, test_data = data.train_val_test_split(val_ratio=0.15, test_ratio=0.15)\n",
    "# max_node_num = max(torch.cat([data.dst,data.src]))+1\n",
    "# max_node_num = data.num_nodes+1\n",
    "max_node_num = 262626  # +1\n",
    "# min_dst_idx, max_dst_idx = int(data.dst.min()), int(data.dst.max())\n",
    "min_dst_idx, max_dst_idx = 0, max_node_num\n",
    "neighbor_loader = LastNeighborLoader(max_node_num, size=20, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GraphAttentionEmbedding(torch.nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, msg_dim, time_enc):\n",
    "        super(GraphAttentionEmbedding, self).__init__()\n",
    "        self.time_enc = time_enc\n",
    "        edge_dim = msg_dim + time_enc.out_channels\n",
    "        self.conv = TransformerConv(in_channels, out_channels, heads=8,\n",
    "                                    dropout=0.0, edge_dim=edge_dim)\n",
    "        self.conv2 = TransformerConv(out_channels*8, out_channels,heads=1, concat=False,\n",
    "                             dropout=0.0, edge_dim=edge_dim)\n",
    "\n",
    "    def forward(self, x, last_update, edge_index, t, msg):\n",
    "        last_update.to(device)\n",
    "        x = x.to(device)\n",
    "        t = t.to(device)\n",
    "        rel_t = last_update[edge_index[0]] - t\n",
    "        rel_t_enc = self.time_enc(rel_t.to(x.dtype))\n",
    "        edge_attr = torch.cat([rel_t_enc, msg], dim=-1)\n",
    "        x = F.relu(self.conv(x, edge_index, edge_attr))\n",
    "        x = F.relu(self.conv2(x, edge_index, edge_attr))\n",
    "        return x\n",
    "\n",
    "\n",
    "class LinkPredictor(torch.nn.Module):\n",
    "    def __init__(self, in_channels):\n",
    "        super(LinkPredictor, self).__init__()\n",
    "        self.lin_src = Linear(in_channels, in_channels*2)\n",
    "        self.lin_dst = Linear(in_channels, in_channels*2)\n",
    "        \n",
    "        self.lin_seq = nn.Sequential(\n",
    "            \n",
    "            Linear(in_channels*4, in_channels*8),\n",
    "            torch.nn.Dropout(0.5),\n",
    "            nn.Tanh(),\n",
    "            Linear(in_channels*8, in_channels*2),\n",
    "            torch.nn.Dropout(0.5),\n",
    "            nn.Tanh(),\n",
    "            Linear(in_channels*2, int(in_channels//2)),\n",
    "            torch.nn.Dropout(0.5),\n",
    "            nn.Tanh(),\n",
    "            Linear(int(in_channels//2), train_data.msg.shape[1]-32)                   \n",
    "        )\n",
    "        \n",
    "\n",
    "    def forward(self, z_src, z_dst):\n",
    "        h = torch.cat([self.lin_src(z_src) , self.lin_dst(z_dst)],dim=-1)      \n",
    "         \n",
    "        h = self.lin_seq (h)\n",
    "        \n",
    "        return h\n",
    "\n",
    "memory_dim = 100         # node state\n",
    "time_dim = 100\n",
    "embedding_dim = 200      # edge embedding\n",
    "\n",
    "memory = TGNMemory(\n",
    "    max_node_num,\n",
    "    train_data.msg.size(-1),\n",
    "    memory_dim,\n",
    "    time_dim,\n",
    "    message_module=IdentityMessage(train_data.msg.size(-1), memory_dim, time_dim),\n",
    "    aggregator_module=LastAggregator(),\n",
    ").to(device)\n",
    "\n",
    "gnn = GraphAttentionEmbedding(\n",
    "    in_channels=memory_dim,\n",
    "    out_channels=embedding_dim,\n",
    "    msg_dim=train_data.msg.size(-1),\n",
    "    time_enc=memory.time_enc,\n",
    ").to(device)\n",
    "\n",
    "link_pred = LinkPredictor(in_channels=embedding_dim).to(device)\n",
    "\n",
    "optimizer = torch.optim.Adam(\n",
    "    set(memory.parameters()) | set(gnn.parameters())\n",
    "    | set(link_pred.parameters()), lr=0.00005, eps=1e-08,weight_decay=0.01)\n",
    "\n",
    "\n",
    "# scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# Helper vector to map global node indices to local ones.\n",
    "assoc = torch.empty(max_node_num, dtype=torch.long, device=device)\n",
    "\n",
    "saved_nodes=set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH=1024\n",
    "def train(train_data):\n",
    "\n",
    "    \n",
    "    memory.train()\n",
    "    gnn.train()\n",
    "    link_pred.train()\n",
    "\n",
    "    memory.reset_state()  # Start with a fresh memory.\n",
    "    neighbor_loader.reset_state()  # Start with an empty graph.\n",
    "    saved_nodes=set()\n",
    "\n",
    "    total_loss = 0\n",
    "    \n",
    "#     print(\"train_before_stage_data:\",train_data)\n",
    "    for batch in train_data.seq_batches(batch_size=BATCH):\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        src, pos_dst, t, msg = batch.src, batch.dst, batch.t, batch.msg        \n",
    "        \n",
    "        n_id = torch.cat([src, pos_dst]).unique()\n",
    "#         n_id = torch.cat([src, pos_dst, neg_src, neg_dst]).unique()\n",
    "        n_id, edge_index, e_id = neighbor_loader(n_id)\n",
    "        assoc[n_id] = torch.arange(n_id.size(0), device=device)\n",
    "\n",
    "        # Get updated memory of all nodes involved in the computation.\n",
    "        z, last_update = memory(n_id)\n",
    "      \n",
    "        z = gnn(z, last_update, edge_index, train_data.t[e_id], train_data.msg[e_id])\n",
    "        \n",
    "        pos_out = link_pred(z[assoc[src]], z[assoc[pos_dst]])       \n",
    "\n",
    "        y_pred = torch.cat([pos_out], dim=0)\n",
    "        \n",
    "#         y_true = torch.cat([torch.zeros(pos_out.size(0),1),torch.ones(neg_out.size(0),1)], dim=0)#\n",
    "        y_true=[]\n",
    "        for m in msg:\n",
    "            l=tensor_find(m[16:-16],1)-1\n",
    "            y_true.append(l)           \n",
    "          \n",
    "        y_true = torch.tensor(y_true).to(device=device)\n",
    "        y_true=y_true.reshape(-1).to(torch.long).to(device=device)\n",
    "        \n",
    "        loss = criterion(y_pred, y_true)\n",
    "        \n",
    "#         loss = criterion(pos_out, torch.ones_like(pos_out))\n",
    "#         loss += criterion(neg_out, torch.zeros_like(neg_out))\n",
    "\n",
    "        # Update memory and neighbor loader with ground-truth state.\n",
    "        memory.update_state(src, pos_dst, t, msg)\n",
    "        neighbor_loader.insert(src, pos_dst)\n",
    "        \n",
    "#         for i in range(len(src)):\n",
    "#             saved_nodes.add(int(src[i]))\n",
    "#             saved_nodes.add(int(pos_dst[i]))\n",
    "\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        memory.detach()\n",
    "#         print(z.shape)\n",
    "        total_loss += float(loss) * batch.num_events\n",
    "#     print(\"trained_stage_data:\",train_data)\n",
    "    return total_loss / train_data.num_events\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|                                                                                                   | 0/30 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 01, Loss: 0.6433\n",
      "  Epoch: 01, Loss: 0.2605\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  3%|██▉                                                                                     | 1/30 [02:45<1:20:13, 165.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 01, Loss: 0.3935\n",
      "  Epoch: 02, Loss: 0.3017\n",
      "  Epoch: 02, Loss: 0.2403\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  7%|█████▊                                                                                  | 2/30 [05:32<1:17:29, 166.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 02, Loss: 0.3789\n",
      "  Epoch: 03, Loss: 0.2957\n",
      "  Epoch: 03, Loss: 0.2422\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|████████▊                                                                               | 3/30 [08:18<1:14:43, 166.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 03, Loss: 0.3823\n",
      "  Epoch: 04, Loss: 0.2921\n",
      "  Epoch: 04, Loss: 0.2342\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 13%|███████████▋                                                                            | 4/30 [11:04<1:11:57, 166.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 04, Loss: 0.3777\n",
      "  Epoch: 05, Loss: 0.2905\n",
      "  Epoch: 05, Loss: 0.2351\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 17%|██████████████▋                                                                         | 5/30 [13:50<1:09:15, 166.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 05, Loss: 0.3803\n",
      "  Epoch: 06, Loss: 0.2854\n",
      "  Epoch: 06, Loss: 0.2328\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|█████████████████▌                                                                      | 6/30 [16:37<1:06:33, 166.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 06, Loss: 0.3688\n",
      "  Epoch: 07, Loss: 0.2849\n",
      "  Epoch: 07, Loss: 0.2327\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 23%|████████████████████▌                                                                   | 7/30 [19:23<1:03:46, 166.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 07, Loss: 0.3686\n",
      "  Epoch: 08, Loss: 0.2804\n",
      "  Epoch: 08, Loss: 0.2308\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 27%|███████████████████████▍                                                                | 8/30 [22:09<1:00:58, 166.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 08, Loss: 0.3703\n",
      "  Epoch: 09, Loss: 0.2786\n",
      "  Epoch: 09, Loss: 0.2265\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 30%|███████████████████████████                                                               | 9/30 [24:56<58:11, 166.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 09, Loss: 0.3796\n",
      "  Epoch: 10, Loss: 0.2837\n",
      "  Epoch: 10, Loss: 0.2368\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 33%|█████████████████████████████▋                                                           | 10/30 [27:41<55:23, 166.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 10, Loss: 0.3707\n",
      "  Epoch: 11, Loss: 0.2826\n",
      "  Epoch: 11, Loss: 0.2259\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 37%|████████████████████████████████▋                                                        | 11/30 [30:28<52:37, 166.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 11, Loss: 0.3702\n",
      "  Epoch: 12, Loss: 0.2762\n",
      "  Epoch: 12, Loss: 0.2240\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|███████████████████████████████████▌                                                     | 12/30 [33:14<49:51, 166.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 12, Loss: 0.3704\n",
      "  Epoch: 13, Loss: 0.2774\n",
      "  Epoch: 13, Loss: 0.2271\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 43%|██████████████████████████████████████▌                                                  | 13/30 [36:00<47:07, 166.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 13, Loss: 0.3722\n",
      "  Epoch: 14, Loss: 0.2742\n",
      "  Epoch: 14, Loss: 0.2314\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 47%|█████████████████████████████████████████▌                                               | 14/30 [38:47<44:21, 166.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 14, Loss: 0.3725\n",
      "  Epoch: 15, Loss: 0.2787\n",
      "  Epoch: 15, Loss: 0.2283\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|████████████████████████████████████████████▌                                            | 15/30 [41:33<41:36, 166.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 15, Loss: 0.3680\n",
      "  Epoch: 16, Loss: 0.2865\n",
      "  Epoch: 16, Loss: 0.2263\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 53%|███████████████████████████████████████████████▍                                         | 16/30 [44:20<38:48, 166.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 16, Loss: 0.3697\n",
      "  Epoch: 17, Loss: 0.2863\n",
      "  Epoch: 17, Loss: 0.2241\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 57%|██████████████████████████████████████████████████▍                                      | 17/30 [47:06<36:01, 166.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 17, Loss: 0.3650\n",
      "  Epoch: 18, Loss: 0.2805\n",
      "  Epoch: 18, Loss: 0.2286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|█████████████████████████████████████████████████████▍                                   | 18/30 [49:52<33:15, 166.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 18, Loss: 0.3647\n",
      "  Epoch: 19, Loss: 0.2843\n",
      "  Epoch: 19, Loss: 0.2317\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 63%|████████████████████████████████████████████████████████▎                                | 19/30 [52:39<30:30, 166.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 19, Loss: 0.3593\n",
      "  Epoch: 20, Loss: 0.2745\n",
      "  Epoch: 20, Loss: 0.2267\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 67%|███████████████████████████████████████████████████████████▎                             | 20/30 [55:25<27:43, 166.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 20, Loss: 0.3696\n",
      "  Epoch: 21, Loss: 0.2798\n",
      "  Epoch: 21, Loss: 0.2297\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 70%|██████████████████████████████████████████████████████████████▎                          | 21/30 [58:11<24:56, 166.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 21, Loss: 0.3661\n",
      "  Epoch: 22, Loss: 0.2771\n",
      "  Epoch: 22, Loss: 0.2247\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 73%|███████████████████████████████████████████████████████████████▊                       | 22/30 [1:00:57<22:09, 166.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 22, Loss: 0.3643\n",
      "  Epoch: 23, Loss: 0.2843\n",
      "  Epoch: 23, Loss: 0.2244\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 77%|██████████████████████████████████████████████████████████████████▋                    | 23/30 [1:03:43<19:23, 166.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 23, Loss: 0.3634\n",
      "  Epoch: 24, Loss: 0.2828\n",
      "  Epoch: 24, Loss: 0.2319\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 80%|█████████████████████████████████████████████████████████████████████▌                 | 24/30 [1:06:29<16:36, 166.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 24, Loss: 0.3729\n",
      "  Epoch: 25, Loss: 0.2873\n",
      "  Epoch: 25, Loss: 0.2224\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 83%|████████████████████████████████████████████████████████████████████████▌              | 25/30 [1:09:16<13:50, 166.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 25, Loss: 0.3713\n",
      "  Epoch: 26, Loss: 0.2773\n",
      "  Epoch: 26, Loss: 0.2285\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 87%|███████████████████████████████████████████████████████████████████████████▍           | 26/30 [1:12:02<11:04, 166.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 26, Loss: 0.3690\n",
      "  Epoch: 27, Loss: 0.2794\n",
      "  Epoch: 27, Loss: 0.2277\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 90%|██████████████████████████████████████████████████████████████████████████████▎        | 27/30 [1:14:48<08:18, 166.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 27, Loss: 0.3727\n",
      "  Epoch: 28, Loss: 0.2786\n",
      "  Epoch: 28, Loss: 0.2241\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 93%|█████████████████████████████████████████████████████████████████████████████████▏     | 28/30 [1:17:34<05:32, 166.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 28, Loss: 0.3745\n",
      "  Epoch: 29, Loss: 0.2742\n",
      "  Epoch: 29, Loss: 0.2250\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 97%|████████████████████████████████████████████████████████████████████████████████████   | 29/30 [1:20:20<02:46, 166.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 29, Loss: 0.3677\n",
      "  Epoch: 30, Loss: 0.2737\n",
      "  Epoch: 30, Loss: 0.2240\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████| 30/30 [1:23:07<00:00, 166.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Epoch: 30, Loss: 0.3640\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "train_graphs=[graph_5_8,graph_5_9,graph_5_11]\n",
    "\n",
    "for epoch in tqdm(range(1, 31)):\n",
    "    for g in train_graphs:\n",
    "        loss = train(g)\n",
    "        print(f'  Epoch: {epoch:02d}, Loss: {loss:.4f}')\n",
    "#     scheduler.step()\n",
    "\n",
    "model=[memory,gnn, link_pred,neighbor_loader]\n",
    "os.system(\"mkdir -p ./models/\")\n",
    "torch.save(model,\"./models/model_saved_share.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_saved_share.pt\r\n"
     ]
    }
   ],
   "source": [
    "ls models/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate the reconstruction results of every day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time \n",
    "\n",
    "@torch.no_grad()\n",
    "def test_day_new(inference_data,path):\n",
    "    if os.path.exists(path):\n",
    "        pass\n",
    "    else:\n",
    "        os.mkdir(path)\n",
    "    \n",
    "    memory.eval()\n",
    "    gnn.eval()\n",
    "    link_pred.eval()\n",
    "    \n",
    "    memory.reset_state()  # Start with a fresh memory. \n",
    "    neighbor_loader.reset_state()  # Start with an empty graph.\n",
    "    \n",
    "    time_with_loss={} # key: time，  value： the losses\n",
    "    total_loss = 0    \n",
    "    edge_list=[]\n",
    "    \n",
    "    unique_nodes=torch.tensor([]).to(device=device)\n",
    "    total_edges=0\n",
    "\n",
    "\n",
    "    start_time=inference_data.t[0]\n",
    "    event_count=0\n",
    "    \n",
    "    pos_o=[]\n",
    "    \n",
    "    loss_list=[]\n",
    "    \n",
    "\n",
    "    print(\"after merge:\",inference_data)\n",
    "    \n",
    "    # Record the running time to evaluate the performance\n",
    "    start = time.perf_counter()\n",
    "\n",
    "    for batch in inference_data.seq_batches(batch_size=BATCH):\n",
    "        \n",
    "        src, pos_dst, t, msg = batch.src, batch.dst, batch.t, batch.msg\n",
    "        unique_nodes=torch.cat([unique_nodes,src,pos_dst]).unique()\n",
    "        total_edges+=BATCH\n",
    "        \n",
    "       \n",
    "        n_id = torch.cat([src, pos_dst]).unique()       \n",
    "        n_id, edge_index, e_id = neighbor_loader(n_id)\n",
    "        assoc[n_id] = torch.arange(n_id.size(0), device=device)\n",
    "\n",
    "        z, last_update = memory(n_id)\n",
    "        z = gnn(z, last_update, edge_index, inference_data.t[e_id], inference_data.msg[e_id])\n",
    "\n",
    "        pos_out = link_pred(z[assoc[src]], z[assoc[pos_dst]])\n",
    "        \n",
    "        pos_o.append(pos_out)\n",
    "        y_pred = torch.cat([pos_out], dim=0)\n",
    "#         y_true = torch.cat(\n",
    "#             [torch.ones(pos_out.size(0))], dim=0).to(torch.long)     \n",
    "#         y_true=y_true.reshape(-1).to(torch.long)\n",
    "\n",
    "        y_true=[]\n",
    "        for m in msg:\n",
    "            l=tensor_find(m[16:-16],1)-1\n",
    "            y_true.append(l) \n",
    "        y_true = torch.tensor(y_true).to(device=device)\n",
    "        y_true=y_true.reshape(-1).to(torch.long).to(device=device)\n",
    "\n",
    "        # Only consider which edge hasn't been correctly predicted.\n",
    "        # For benign graphs, the behaviors patterns are similar and therefore their losses are small\n",
    "        # For anoamlous behaviors, some behaviors might not be seen before, so the probability of predicting those edges are low. Thus their losses are high.\n",
    "        loss = criterion(y_pred, y_true)\n",
    "\n",
    "        total_loss += float(loss) * batch.num_events\n",
    "     \n",
    "        \n",
    "        # update the edges in the batch to the memory and neighbor_loader\n",
    "        memory.update_state(src, pos_dst, t, msg)\n",
    "        neighbor_loader.insert(src, pos_dst)\n",
    "        \n",
    "        # compute the loss for each edge\n",
    "        each_edge_loss= cal_pos_edges_loss_multiclass(pos_out,y_true)\n",
    "        \n",
    "        for i in range(len(pos_out)):\n",
    "            srcnode=int(src[i])\n",
    "            dstnode=int(pos_dst[i])  \n",
    "            \n",
    "            srcmsg=str(nodeid2msg[srcnode]) \n",
    "            dstmsg=str(nodeid2msg[dstnode])\n",
    "            t_var=int(t[i])\n",
    "            edgeindex=tensor_find(msg[i][16:-16],1)   \n",
    "            edge_type=rel2id[edgeindex]\n",
    "            loss=each_edge_loss[i]    \n",
    "\n",
    "            temp_dic={}\n",
    "            temp_dic['loss']=float(loss)\n",
    "            temp_dic['srcnode']=srcnode\n",
    "            temp_dic['dstnode']=dstnode\n",
    "            temp_dic['srcmsg']=srcmsg\n",
    "            temp_dic['dstmsg']=dstmsg\n",
    "            temp_dic['edge_type']=edge_type\n",
    "            temp_dic['time']=t_var\n",
    "            \n",
    "\n",
    "#             if \"netflow\" in srcmsg or \"netflow\" in dstmsg:\n",
    "#                 temp_dic['loss']=0\n",
    "            edge_list.append(temp_dic)\n",
    "        \n",
    "        event_count+=len(batch.src)\n",
    "        if t[-1]>start_time+60000000000*15:\n",
    "            # Here is a checkpoint, which records all edge losses in the current time window\n",
    "#             loss=total_loss/event_count\n",
    "            time_interval=ns_time_to_datetime_US(start_time)+\"~\"+ns_time_to_datetime_US(t[-1])\n",
    "\n",
    "            end = time.perf_counter()\n",
    "            time_with_loss[time_interval]={'loss':loss,\n",
    "                                \n",
    "                                          'nodes_count':len(unique_nodes),\n",
    "                                          'total_edges':total_edges,\n",
    "                                          'costed_time':(end-start)}\n",
    "            \n",
    "            \n",
    "            log=open(path+\"/\"+time_interval+\".txt\",'w')\n",
    "            \n",
    "            for e in edge_list: \n",
    "#                 temp_key=e['srcmsg']+e['dstmsg']+e['edge_type']\n",
    "#                 if temp_key in train_edge_set:      \n",
    "# #                     e['loss']=(e['loss']-train_edge_set[temp_key]) if e['loss']>=train_edge_set[temp_key] else 0  \n",
    "# #                     e['loss']=abs(e['loss']-train_edge_set[temp_key])\n",
    "                    \n",
    "#                     e['modified']=True\n",
    "#                 else:\n",
    "#                     e['modified']=False\n",
    "                loss+=e['loss']\n",
    "\n",
    "            loss=loss/event_count   \n",
    "            print(f'Time: {time_interval}, Loss: {loss:.4f}, Nodes_count: {len(unique_nodes)}, Cost Time: {(end-start):.2f}s')\n",
    "            edge_list = sorted(edge_list, key=lambda x:x['loss'],reverse=True)   # Rank the results based on edge losses\n",
    "            for e in edge_list: \n",
    "                log.write(str(e))\n",
    "                log.write(\"\\n\") \n",
    "            event_count=0\n",
    "            total_loss=0\n",
    "            loss=0\n",
    "            start_time=t[-1]\n",
    "            log.close()\n",
    "            edge_list.clear()\n",
    "            \n",
    " \n",
    "    return time_with_loss\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_5_12=torch.load(\"./train_graph/graph_5_12.TemporalData.simple\").to(device=device)\n",
    "graph_5_15=torch.load(\"./train_graph/graph_5_15.TemporalData.simple\").to(device=device)\n",
    "graph_5_16=torch.load(\"./train_graph/graph_5_16.TemporalData.simple\").to(device=device)\n",
    "graph_5_17=torch.load(\"./train_graph/graph_5_17.TemporalData.simple\").to(device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model=torch.load(\"./models/model_saved_share.pt\")\n",
    "memory,gnn, link_pred,neighbor_loader=model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after merge: TemporalData(dst=[1397931], msg=[1397931, 41], src=[1397931], t=[1397931])\n",
      "Time: 2019-05-08 00:00:00.008989193~2019-05-08 00:15:12.819738250, Loss: 0.5694, Nodes_count: 956, Cost Time: 1.11s\n",
      "Time: 2019-05-08 00:15:12.819738250~2019-05-08 00:34:12.708825175, Loss: 0.3083, Nodes_count: 1304, Cost Time: 1.79s\n",
      "Time: 2019-05-08 00:34:12.708825175~2019-05-08 00:52:03.918713514, Loss: 0.5421, Nodes_count: 1655, Cost Time: 2.33s\n",
      "Time: 2019-05-08 00:52:03.918713514~2019-05-08 01:07:12.359122777, Loss: 0.4147, Nodes_count: 2081, Cost Time: 2.98s\n",
      "Time: 2019-05-08 01:07:12.359122777~2019-05-08 01:25:38.818821035, Loss: 0.2554, Nodes_count: 2276, Cost Time: 3.41s\n",
      "Time: 2019-05-08 01:25:38.818821035~2019-05-08 01:41:28.558761306, Loss: 0.2894, Nodes_count: 2443, Cost Time: 3.88s\n",
      "Time: 2019-05-08 01:41:28.558761306~2019-05-08 01:58:50.438624115, Loss: 0.3976, Nodes_count: 2721, Cost Time: 4.69s\n",
      "Time: 2019-05-08 01:58:50.438624115~2019-05-08 02:14:03.438612575, Loss: 0.5095, Nodes_count: 2831, Cost Time: 6.58s\n",
      "Time: 2019-05-08 02:14:03.438612575~2019-05-08 02:31:29.678815724, Loss: 0.4814, Nodes_count: 2888, Cost Time: 7.01s\n",
      "Time: 2019-05-08 02:31:29.678815724~2019-05-08 02:49:11.628571232, Loss: 0.3404, Nodes_count: 2981, Cost Time: 7.26s\n",
      "Time: 2019-05-08 02:49:11.628571232~2019-05-08 03:05:00.009532426, Loss: 0.3031, Nodes_count: 3137, Cost Time: 8.38s\n",
      "Time: 2019-05-08 03:05:00.009532426~2019-05-08 03:22:10.148619540, Loss: 0.4827, Nodes_count: 3199, Cost Time: 8.78s\n",
      "Time: 2019-05-08 03:22:10.148619540~2019-05-08 03:37:34.308743449, Loss: 0.5102, Nodes_count: 3352, Cost Time: 9.96s\n",
      "Time: 2019-05-08 03:37:34.308743449~2019-05-08 03:54:26.068445630, Loss: 0.6412, Nodes_count: 3373, Cost Time: 10.36s\n",
      "Time: 2019-05-08 03:54:26.068445630~2019-05-08 04:14:38.858533697, Loss: 0.8433, Nodes_count: 3434, Cost Time: 11.22s\n",
      "Time: 2019-05-08 04:14:38.858533697~2019-05-08 04:30:43.638608979, Loss: 0.4342, Nodes_count: 3523, Cost Time: 11.66s\n",
      "Time: 2019-05-08 04:30:43.638608979~2019-05-08 06:11:00.006878784, Loss: 0.5929, Nodes_count: 3526, Cost Time: 11.74s\n",
      "Time: 2019-05-08 06:11:00.006878784~2019-05-08 08:11:00.066513872, Loss: 0.8043, Nodes_count: 3527, Cost Time: 11.80s\n",
      "Time: 2019-05-08 08:11:00.066513872~2019-05-08 09:01:03.038121843, Loss: 0.4973, Nodes_count: 3542, Cost Time: 11.86s\n",
      "Time: 2019-05-08 09:01:03.038121843~2019-05-08 09:16:32.689081876, Loss: 0.5141, Nodes_count: 3691, Cost Time: 12.49s\n",
      "Time: 2019-05-08 09:16:32.689081876~2019-05-08 09:31:52.598135918, Loss: 0.3855, Nodes_count: 3837, Cost Time: 13.14s\n",
      "Time: 2019-05-08 09:31:52.598135918~2019-05-08 09:46:53.479321103, Loss: 0.2683, Nodes_count: 3944, Cost Time: 13.63s\n",
      "Time: 2019-05-08 09:46:53.479321103~2019-05-08 10:02:49.268953616, Loss: 0.3557, Nodes_count: 4121, Cost Time: 14.82s\n",
      "Time: 2019-05-08 10:02:49.268953616~2019-05-08 10:18:17.839131951, Loss: 0.2407, Nodes_count: 4451, Cost Time: 16.38s\n",
      "Time: 2019-05-08 10:18:17.839131951~2019-05-08 10:33:18.328090790, Loss: 0.3588, Nodes_count: 4680, Cost Time: 17.30s\n",
      "Time: 2019-05-08 10:33:18.328090790~2019-05-08 10:48:37.447921283, Loss: 0.4539, Nodes_count: 4899, Cost Time: 19.28s\n",
      "Time: 2019-05-08 10:48:37.447921283~2019-05-08 11:03:57.958110787, Loss: 0.3531, Nodes_count: 5273, Cost Time: 20.73s\n",
      "Time: 2019-05-08 11:03:57.958110787~2019-05-08 11:20:30.669254956, Loss: 0.4941, Nodes_count: 5540, Cost Time: 22.90s\n",
      "Time: 2019-05-08 11:20:30.669254956~2019-05-08 11:37:46.138261490, Loss: 0.7981, Nodes_count: 5890, Cost Time: 24.57s\n",
      "Time: 2019-05-08 11:37:46.138261490~2019-05-08 11:53:13.428961311, Loss: 0.3894, Nodes_count: 6051, Cost Time: 26.31s\n",
      "Time: 2019-05-08 11:53:13.428961311~2019-05-08 12:08:48.249011424, Loss: 0.4836, Nodes_count: 6300, Cost Time: 27.64s\n",
      "Time: 2019-05-08 12:08:48.249011424~2019-05-08 12:24:12.377925827, Loss: 0.4011, Nodes_count: 6555, Cost Time: 28.98s\n",
      "Time: 2019-05-08 12:24:12.377925827~2019-05-08 12:39:33.618151463, Loss: 0.2917, Nodes_count: 7064, Cost Time: 34.96s\n",
      "Time: 2019-05-08 12:39:33.618151463~2019-05-08 12:54:47.168052243, Loss: 0.8167, Nodes_count: 7264, Cost Time: 39.02s\n",
      "Time: 2019-05-08 12:54:47.168052243~2019-05-08 13:09:52.556232093, Loss: 0.5011, Nodes_count: 7407, Cost Time: 40.60s\n",
      "Time: 2019-05-08 13:09:52.556232093~2019-05-08 13:25:00.007840851, Loss: 0.4785, Nodes_count: 7629, Cost Time: 42.34s\n",
      "Time: 2019-05-08 13:25:00.007840851~2019-05-08 13:40:16.536404006, Loss: 0.4619, Nodes_count: 7770, Cost Time: 43.81s\n",
      "Time: 2019-05-08 13:40:16.536404006~2019-05-08 13:55:29.098836108, Loss: 0.6340, Nodes_count: 7948, Cost Time: 45.32s\n",
      "Time: 2019-05-08 13:55:29.098836108~2019-05-08 14:10:38.556117583, Loss: 0.8108, Nodes_count: 8091, Cost Time: 47.77s\n",
      "Time: 2019-05-08 14:10:38.556117583~2019-05-08 14:26:17.338506099, Loss: 0.6738, Nodes_count: 8250, Cost Time: 49.00s\n",
      "Time: 2019-05-08 14:26:17.338506099~2019-05-08 14:41:59.588590808, Loss: 0.5067, Nodes_count: 8449, Cost Time: 50.40s\n",
      "Time: 2019-05-08 14:41:59.588590808~2019-05-08 14:57:02.998905844, Loss: 0.4334, Nodes_count: 8636, Cost Time: 51.98s\n",
      "Time: 2019-05-08 14:57:02.998905844~2019-05-08 15:12:08.576004726, Loss: 0.6244, Nodes_count: 8858, Cost Time: 53.24s\n",
      "Time: 2019-05-08 15:12:08.576004726~2019-05-08 15:27:37.817602541, Loss: 0.6241, Nodes_count: 9053, Cost Time: 54.64s\n",
      "Time: 2019-05-08 15:27:37.817602541~2019-05-08 15:43:16.616310520, Loss: 0.4177, Nodes_count: 9250, Cost Time: 56.37s\n",
      "Time: 2019-05-08 15:43:16.616310520~2019-05-08 15:59:01.447578163, Loss: 0.4098, Nodes_count: 9367, Cost Time: 58.23s\n",
      "Time: 2019-05-08 15:59:01.447578163~2019-05-08 16:15:28.545870096, Loss: 0.4818, Nodes_count: 9625, Cost Time: 61.46s\n",
      "Time: 2019-05-08 16:15:28.545870096~2019-05-08 16:30:53.157609683, Loss: 0.5090, Nodes_count: 9781, Cost Time: 63.01s\n",
      "Time: 2019-05-08 16:30:53.157609683~2019-05-08 16:46:58.555895559, Loss: 0.4761, Nodes_count: 9813, Cost Time: 63.70s\n",
      "Time: 2019-05-08 16:46:58.555895559~2019-05-08 17:02:02.545793981, Loss: 0.0463, Nodes_count: 9814, Cost Time: 64.13s\n",
      "Time: 2019-05-08 17:02:02.545793981~2019-05-08 17:18:16.555854536, Loss: 0.0416, Nodes_count: 9814, Cost Time: 64.59s\n",
      "Time: 2019-05-08 17:18:16.555854536~2019-05-08 17:33:24.545754239, Loss: 0.0337, Nodes_count: 9814, Cost Time: 65.01s\n",
      "Time: 2019-05-08 17:33:24.545754239~2019-05-08 17:48:40.536052658, Loss: 0.0359, Nodes_count: 9816, Cost Time: 65.41s\n",
      "Time: 2019-05-08 17:48:40.536052658~2019-05-08 18:03:42.535879500, Loss: 0.0384, Nodes_count: 9816, Cost Time: 65.82s\n",
      "Time: 2019-05-08 18:03:42.535879500~2019-05-08 18:20:08.545852982, Loss: 0.0301, Nodes_count: 9817, Cost Time: 66.28s\n",
      "Time: 2019-05-08 18:20:08.545852982~2019-05-08 18:35:26.565719693, Loss: 0.0284, Nodes_count: 9817, Cost Time: 66.70s\n",
      "Time: 2019-05-08 18:35:26.565719693~2019-05-08 18:50:52.555722324, Loss: 0.0285, Nodes_count: 9818, Cost Time: 67.10s\n",
      "Time: 2019-05-08 18:50:52.555722324~2019-05-08 19:06:54.555724938, Loss: 0.0297, Nodes_count: 9818, Cost Time: 67.57s\n",
      "Time: 2019-05-08 19:06:54.555724938~2019-05-08 19:22:06.545763631, Loss: 0.0282, Nodes_count: 9820, Cost Time: 67.98s\n",
      "Time: 2019-05-08 19:22:06.545763631~2019-05-08 19:37:36.555680158, Loss: 0.0301, Nodes_count: 9822, Cost Time: 68.39s\n",
      "Time: 2019-05-08 19:37:36.555680158~2019-05-08 19:53:08.555634102, Loss: 0.0313, Nodes_count: 9823, Cost Time: 68.81s\n",
      "Time: 2019-05-08 19:53:08.555634102~2019-05-08 20:09:18.535858004, Loss: 0.0400, Nodes_count: 9825, Cost Time: 69.27s\n",
      "Time: 2019-05-08 20:09:18.535858004~2019-05-08 20:24:38.555613914, Loss: 0.0365, Nodes_count: 9825, Cost Time: 69.69s\n",
      "Time: 2019-05-08 20:24:38.555613914~2019-05-08 20:39:50.555567844, Loss: 0.0353, Nodes_count: 9826, Cost Time: 70.09s\n",
      "Time: 2019-05-08 20:39:50.555567844~2019-05-08 20:55:00.615720938, Loss: 0.0418, Nodes_count: 9826, Cost Time: 70.50s\n",
      "Time: 2019-05-08 20:55:00.615720938~2019-05-08 21:11:00.685577435, Loss: 0.0484, Nodes_count: 9827, Cost Time: 70.97s\n",
      "Time: 2019-05-08 21:11:00.685577435~2019-05-08 21:26:16.555510023, Loss: 0.0406, Nodes_count: 9828, Cost Time: 71.38s\n",
      "Time: 2019-05-08 21:26:16.555510023~2019-05-08 21:41:32.565556712, Loss: 0.0343, Nodes_count: 9829, Cost Time: 71.79s\n",
      "Time: 2019-05-08 21:41:32.565556712~2019-05-08 21:56:50.555482977, Loss: 0.0343, Nodes_count: 9829, Cost Time: 72.20s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time: 2019-05-08 21:56:50.555482977~2019-05-08 22:13:04.535675246, Loss: 0.0363, Nodes_count: 9830, Cost Time: 72.66s\n",
      "Time: 2019-05-08 22:13:04.535675246~2019-05-08 22:28:22.555416040, Loss: 0.0343, Nodes_count: 9830, Cost Time: 73.07s\n",
      "Time: 2019-05-08 22:28:22.555416040~2019-05-08 22:43:48.565468446, Loss: 0.0306, Nodes_count: 9830, Cost Time: 73.48s\n",
      "Time: 2019-05-08 22:43:48.565468446~2019-05-08 22:59:12.555370285, Loss: 0.0281, Nodes_count: 9831, Cost Time: 73.89s\n",
      "Time: 2019-05-08 22:59:12.555370285~2019-05-08 23:16:02.545423562, Loss: 0.2610, Nodes_count: 10591, Cost Time: 77.34s\n",
      "Time: 2019-05-08 23:16:02.545423562~2019-05-08 23:32:10.545375783, Loss: 0.2203, Nodes_count: 11174, Cost Time: 78.64s\n",
      "Time: 2019-05-08 23:32:10.545375783~2019-05-08 23:48:40.545322654, Loss: 0.2006, Nodes_count: 11466, Cost Time: 79.59s\n"
     ]
    }
   ],
   "source": [
    "ans_5_8=test_day_new(graph_5_8,\"./graph_5_8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after merge: TemporalData(dst=[7402525], msg=[7402525, 41], src=[7402525], t=[7402525])\n",
      "Time: 2019-05-09 00:00:00.005370573~2019-05-09 00:16:10.545268695, Loss: 0.1944, Nodes_count: 34, Cost Time: 0.43s\n",
      "Time: 2019-05-09 00:16:10.545268695~2019-05-09 00:31:30.555241471, Loss: 0.0307, Nodes_count: 34, Cost Time: 0.84s\n",
      "Time: 2019-05-09 00:31:30.555241471~2019-05-09 00:46:46.555249187, Loss: 0.0377, Nodes_count: 35, Cost Time: 1.24s\n",
      "Time: 2019-05-09 00:46:46.555249187~2019-05-09 01:01:56.555203078, Loss: 0.0426, Nodes_count: 37, Cost Time: 1.65s\n",
      "Time: 2019-05-09 01:01:56.555203078~2019-05-09 01:18:22.555177589, Loss: 0.0345, Nodes_count: 38, Cost Time: 2.11s\n",
      "Time: 2019-05-09 01:18:22.555177589~2019-05-09 01:33:38.555157961, Loss: 0.0384, Nodes_count: 39, Cost Time: 2.54s\n",
      "Time: 2019-05-09 01:33:38.555157961~2019-05-09 01:48:52.555136318, Loss: 0.0378, Nodes_count: 39, Cost Time: 2.95s\n",
      "Time: 2019-05-09 01:48:52.555136318~2019-05-09 02:03:56.555112888, Loss: 0.0500, Nodes_count: 39, Cost Time: 3.38s\n",
      "Time: 2019-05-09 02:03:56.555112888~2019-05-09 02:20:26.565224554, Loss: 0.0475, Nodes_count: 40, Cost Time: 3.84s\n",
      "Time: 2019-05-09 02:20:26.565224554~2019-05-09 02:35:44.555068254, Loss: 0.0493, Nodes_count: 41, Cost Time: 4.26s\n",
      "Time: 2019-05-09 02:35:44.555068254~2019-05-09 02:51:00.555050523, Loss: 0.0518, Nodes_count: 42, Cost Time: 4.67s\n",
      "Time: 2019-05-09 02:51:00.555050523~2019-05-09 03:06:52.555029929, Loss: 0.0474, Nodes_count: 42, Cost Time: 5.13s\n",
      "Time: 2019-05-09 03:06:52.555029929~2019-05-09 03:22:18.555003083, Loss: 0.0482, Nodes_count: 42, Cost Time: 5.55s\n",
      "Time: 2019-05-09 03:22:18.555003083~2019-05-09 03:37:44.545058785, Loss: 0.0496, Nodes_count: 43, Cost Time: 5.97s\n",
      "Time: 2019-05-09 03:37:44.545058785~2019-05-09 03:53:08.554938183, Loss: 0.0458, Nodes_count: 44, Cost Time: 6.40s\n",
      "Time: 2019-05-09 03:53:08.554938183~2019-05-09 04:09:18.545010023, Loss: 0.0408, Nodes_count: 46, Cost Time: 6.86s\n",
      "Time: 2019-05-09 04:09:18.545010023~2019-05-09 04:24:42.575201598, Loss: 0.0403, Nodes_count: 47, Cost Time: 7.27s\n",
      "Time: 2019-05-09 04:24:42.575201598~2019-05-09 04:40:20.575172763, Loss: 0.0427, Nodes_count: 48, Cost Time: 7.68s\n",
      "Time: 2019-05-09 04:40:20.575172763~2019-05-09 04:55:46.544939968, Loss: 0.0418, Nodes_count: 49, Cost Time: 8.08s\n",
      "Time: 2019-05-09 04:55:46.544939968~2019-05-09 05:11:44.554846553, Loss: 0.0360, Nodes_count: 59, Cost Time: 8.55s\n",
      "Time: 2019-05-09 05:11:44.554846553~2019-05-09 05:27:16.544847730, Loss: 0.0318, Nodes_count: 59, Cost Time: 8.96s\n",
      "Time: 2019-05-09 05:27:16.544847730~2019-05-09 05:42:52.554803277, Loss: 0.0326, Nodes_count: 59, Cost Time: 9.36s\n",
      "Time: 2019-05-09 05:42:52.554803277~2019-05-09 05:58:20.554778934, Loss: 0.0331, Nodes_count: 59, Cost Time: 9.77s\n",
      "Time: 2019-05-09 05:58:20.554778934~2019-05-09 06:14:38.544811614, Loss: 0.0312, Nodes_count: 60, Cost Time: 10.23s\n",
      "Time: 2019-05-09 06:14:38.544811614~2019-05-09 06:30:12.564849097, Loss: 0.0296, Nodes_count: 61, Cost Time: 10.64s\n",
      "Time: 2019-05-09 06:30:12.564849097~2019-05-09 06:45:38.554717730, Loss: 0.0287, Nodes_count: 61, Cost Time: 11.05s\n",
      "Time: 2019-05-09 06:45:38.554717730~2019-05-09 07:00:52.554691279, Loss: 0.0307, Nodes_count: 63, Cost Time: 11.45s\n",
      "Time: 2019-05-09 07:00:52.554691279~2019-05-09 07:17:27.874999097, Loss: 0.0323, Nodes_count: 64, Cost Time: 11.92s\n",
      "Time: 2019-05-09 07:17:27.874999097~2019-05-09 07:33:00.026589709, Loss: 0.0304, Nodes_count: 64, Cost Time: 12.32s\n",
      "Time: 2019-05-09 07:33:00.026589709~2019-05-09 07:48:30.544694696, Loss: 0.0314, Nodes_count: 65, Cost Time: 12.73s\n",
      "Time: 2019-05-09 07:48:30.544694696~2019-05-09 08:03:37.044717093, Loss: 0.0335, Nodes_count: 78, Cost Time: 13.14s\n",
      "Time: 2019-05-09 08:03:37.044717093~2019-05-09 08:19:14.554597180, Loss: 0.0710, Nodes_count: 103, Cost Time: 13.60s\n",
      "Time: 2019-05-09 08:19:14.554597180~2019-05-09 08:35:48.554559982, Loss: 0.0674, Nodes_count: 104, Cost Time: 14.06s\n",
      "Time: 2019-05-09 08:35:48.554559982~2019-05-09 08:52:32.554548183, Loss: 0.1801, Nodes_count: 300, Cost Time: 14.91s\n",
      "Time: 2019-05-09 08:52:32.554548183~2019-05-09 09:07:33.176225746, Loss: 0.0467, Nodes_count: 409, Cost Time: 18.04s\n",
      "Time: 2019-05-09 09:07:33.176225746~2019-05-09 09:22:33.847091870, Loss: 0.1300, Nodes_count: 1073, Cost Time: 37.10s\n",
      "Time: 2019-05-09 09:22:33.847091870~2019-05-09 09:37:34.586475376, Loss: 0.2441, Nodes_count: 1662, Cost Time: 51.38s\n",
      "Time: 2019-05-09 09:37:34.586475376~2019-05-09 09:52:36.996216319, Loss: 0.0807, Nodes_count: 1955, Cost Time: 66.29s\n",
      "Time: 2019-05-09 09:52:36.996216319~2019-05-09 10:07:37.446028767, Loss: 0.1881, Nodes_count: 2421, Cost Time: 84.60s\n",
      "Time: 2019-05-09 10:07:37.446028767~2019-05-09 10:22:38.546365532, Loss: 0.1254, Nodes_count: 2752, Cost Time: 110.36s\n",
      "Time: 2019-05-09 10:22:38.546365532~2019-05-09 10:37:41.286336506, Loss: 0.2111, Nodes_count: 3320, Cost Time: 132.94s\n",
      "Time: 2019-05-09 10:37:41.286336506~2019-05-09 10:52:41.806335723, Loss: 0.2413, Nodes_count: 3463, Cost Time: 149.73s\n",
      "Time: 2019-05-09 10:52:41.806335723~2019-05-09 11:07:43.466012856, Loss: 0.1118, Nodes_count: 3610, Cost Time: 162.89s\n",
      "Time: 2019-05-09 11:07:43.466012856~2019-05-09 11:22:49.466199124, Loss: 0.3576, Nodes_count: 3920, Cost Time: 175.86s\n",
      "Time: 2019-05-09 11:22:49.466199124~2019-05-09 11:37:52.625939541, Loss: 0.2871, Nodes_count: 4069, Cost Time: 191.74s\n",
      "Time: 2019-05-09 11:37:52.625939541~2019-05-09 11:52:56.375932380, Loss: 0.4082, Nodes_count: 4375, Cost Time: 204.40s\n",
      "Time: 2019-05-09 11:52:56.375932380~2019-05-09 12:08:10.315899437, Loss: 0.3297, Nodes_count: 4653, Cost Time: 212.92s\n",
      "Time: 2019-05-09 12:08:10.315899437~2019-05-09 12:23:16.154340146, Loss: 0.3095, Nodes_count: 4930, Cost Time: 221.52s\n",
      "Time: 2019-05-09 12:23:16.154340146~2019-05-09 12:38:28.794456861, Loss: 0.3700, Nodes_count: 5266, Cost Time: 226.02s\n",
      "Time: 2019-05-09 12:38:28.794456861~2019-05-09 12:53:30.125977136, Loss: 0.3733, Nodes_count: 5554, Cost Time: 233.86s\n",
      "Time: 2019-05-09 12:53:30.125977136~2019-05-09 13:08:33.864373023, Loss: 0.3513, Nodes_count: 5702, Cost Time: 240.62s\n",
      "Time: 2019-05-09 13:08:33.864373023~2019-05-09 13:23:34.605929519, Loss: 0.3711, Nodes_count: 5938, Cost Time: 248.14s\n",
      "Time: 2019-05-09 13:23:34.605929519~2019-05-09 13:38:38.824276435, Loss: 0.3826, Nodes_count: 6247, Cost Time: 254.01s\n",
      "Time: 2019-05-09 13:38:38.824276435~2019-05-09 13:53:39.854160753, Loss: 0.3433, Nodes_count: 6514, Cost Time: 263.73s\n",
      "Time: 2019-05-09 13:53:39.854160753~2019-05-09 14:08:43.194257996, Loss: 0.2805, Nodes_count: 6736, Cost Time: 270.19s\n",
      "Time: 2019-05-09 14:08:43.194257996~2019-05-09 14:23:43.764424916, Loss: 0.4528, Nodes_count: 6983, Cost Time: 276.77s\n",
      "Time: 2019-05-09 14:23:43.764424916~2019-05-09 14:38:45.355937781, Loss: 0.3663, Nodes_count: 7214, Cost Time: 284.62s\n",
      "Time: 2019-05-09 14:38:45.355937781~2019-05-09 14:53:46.574313430, Loss: 0.3400, Nodes_count: 7474, Cost Time: 292.96s\n",
      "Time: 2019-05-09 14:53:46.574313430~2019-05-09 15:08:47.625781487, Loss: 0.2902, Nodes_count: 7649, Cost Time: 297.05s\n",
      "Time: 2019-05-09 15:08:47.625781487~2019-05-09 15:23:48.584407016, Loss: 0.3895, Nodes_count: 8080, Cost Time: 304.50s\n",
      "Time: 2019-05-09 15:23:48.584407016~2019-05-09 15:38:57.813985226, Loss: 0.4834, Nodes_count: 8269, Cost Time: 313.21s\n",
      "Time: 2019-05-09 15:38:57.813985226~2019-05-09 15:54:00.544003177, Loss: 0.2732, Nodes_count: 8486, Cost Time: 318.52s\n",
      "Time: 2019-05-09 15:54:00.544003177~2019-05-09 16:09:08.304047646, Loss: 0.3723, Nodes_count: 8787, Cost Time: 323.51s\n",
      "Time: 2019-05-09 16:09:08.304047646~2019-05-09 16:24:18.215717971, Loss: 0.3704, Nodes_count: 9130, Cost Time: 331.82s\n",
      "Time: 2019-05-09 16:24:18.215717971~2019-05-09 16:39:19.625802649, Loss: 0.3773, Nodes_count: 9412, Cost Time: 340.58s\n",
      "Time: 2019-05-09 16:39:19.625802649~2019-05-09 16:54:28.644015944, Loss: 0.5345, Nodes_count: 9714, Cost Time: 349.36s\n",
      "Time: 2019-05-09 16:54:28.644015944~2019-05-09 17:09:46.543884325, Loss: 0.3798, Nodes_count: 9881, Cost Time: 355.28s\n",
      "Time: 2019-05-09 17:09:46.543884325~2019-05-09 17:25:08.563944635, Loss: 0.2726, Nodes_count: 10094, Cost Time: 362.69s\n",
      "Time: 2019-05-09 17:25:08.563944635~2019-05-09 17:40:09.345544385, Loss: 0.3235, Nodes_count: 10273, Cost Time: 368.96s\n",
      "Time: 2019-05-09 17:40:09.345544385~2019-05-09 17:55:52.455519795, Loss: 0.2509, Nodes_count: 10405, Cost Time: 373.75s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time: 2019-05-09 17:55:52.455519795~2019-05-09 18:12:16.474050413, Loss: 0.2436, Nodes_count: 10713, Cost Time: 377.74s\n",
      "Time: 2019-05-09 18:12:16.474050413~2019-05-09 18:27:43.863827363, Loss: 0.4219, Nodes_count: 11032, Cost Time: 380.44s\n",
      "Time: 2019-05-09 18:27:43.863827363~2019-05-09 18:43:09.954117708, Loss: 0.4606, Nodes_count: 11311, Cost Time: 381.86s\n",
      "Time: 2019-05-09 18:43:09.954117708~2019-05-09 19:00:54.874007577, Loss: 0.6333, Nodes_count: 11578, Cost Time: 383.99s\n",
      "Time: 2019-05-09 19:00:54.874007577~2019-05-09 19:16:12.306624651, Loss: 0.5619, Nodes_count: 11855, Cost Time: 385.95s\n",
      "Time: 2019-05-09 19:16:12.306624651~2019-05-09 19:31:17.765397125, Loss: 0.3394, Nodes_count: 12154, Cost Time: 387.85s\n",
      "Time: 2019-05-09 19:31:17.765397125~2019-05-09 19:46:24.404104182, Loss: 0.4282, Nodes_count: 12304, Cost Time: 389.35s\n",
      "Time: 2019-05-09 19:46:24.404104182~2019-05-09 20:01:35.753820311, Loss: 0.4143, Nodes_count: 12547, Cost Time: 391.00s\n",
      "Time: 2019-05-09 20:01:35.753820311~2019-05-09 20:17:04.163811336, Loss: 0.4032, Nodes_count: 12773, Cost Time: 392.66s\n",
      "Time: 2019-05-09 20:17:04.163811336~2019-05-09 20:35:01.995579473, Loss: 0.4885, Nodes_count: 12956, Cost Time: 394.38s\n",
      "Time: 2019-05-09 20:35:01.995579473~2019-05-09 20:51:43.095265840, Loss: 0.4362, Nodes_count: 13217, Cost Time: 396.10s\n",
      "Time: 2019-05-09 20:51:43.095265840~2019-05-09 21:07:48.103865238, Loss: 0.4253, Nodes_count: 13477, Cost Time: 397.38s\n",
      "Time: 2019-05-09 21:07:48.103865238~2019-05-09 21:22:58.133520641, Loss: 0.4133, Nodes_count: 13709, Cost Time: 399.14s\n",
      "Time: 2019-05-09 21:22:58.133520641~2019-05-09 21:38:22.693597094, Loss: 0.4961, Nodes_count: 13825, Cost Time: 400.74s\n",
      "Time: 2019-05-09 21:38:22.693597094~2019-05-09 21:58:05.723725222, Loss: 0.4481, Nodes_count: 13995, Cost Time: 401.78s\n",
      "Time: 2019-05-09 21:58:05.723725222~2019-05-09 22:13:08.743472073, Loss: 0.4604, Nodes_count: 14161, Cost Time: 403.38s\n",
      "Time: 2019-05-09 22:13:08.743472073~2019-05-09 22:28:54.596336941, Loss: 0.4434, Nodes_count: 14318, Cost Time: 404.64s\n",
      "Time: 2019-05-09 22:28:54.596336941~2019-05-09 22:44:00.053464293, Loss: 0.5106, Nodes_count: 14541, Cost Time: 406.02s\n",
      "Time: 2019-05-09 22:44:00.053464293~2019-05-09 22:59:07.356054195, Loss: 0.5289, Nodes_count: 14832, Cost Time: 409.35s\n",
      "Time: 2019-05-09 22:59:07.356054195~2019-05-09 23:15:12.385109551, Loss: 0.4835, Nodes_count: 16002, Cost Time: 411.89s\n",
      "Time: 2019-05-09 23:15:12.385109551~2019-05-09 23:32:10.784984219, Loss: 0.5895, Nodes_count: 16282, Cost Time: 414.79s\n",
      "Time: 2019-05-09 23:32:10.784984219~2019-05-09 23:47:42.535086885, Loss: 0.5773, Nodes_count: 17091, Cost Time: 416.75s\n"
     ]
    }
   ],
   "source": [
    "ans_5_9=test_day_new(graph_5_9,\"./graph_5_9\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after merge: TemporalData(dst=[1298877], msg=[1298877, 41], src=[1298877], t=[1298877])\n",
      "Time: 2019-05-11 00:00:00.014349655~2019-05-11 00:15:10.923289891, Loss: 0.7230, Nodes_count: 1018, Cost Time: 0.74s\n",
      "Time: 2019-05-11 00:15:10.923289891~2019-05-11 00:32:03.961727449, Loss: 0.5396, Nodes_count: 1728, Cost Time: 1.81s\n",
      "Time: 2019-05-11 00:32:03.961727449~2019-05-11 00:48:59.682989652, Loss: 0.4649, Nodes_count: 2020, Cost Time: 2.83s\n",
      "Time: 2019-05-11 00:48:59.682989652~2019-05-11 01:06:24.691531973, Loss: 0.3805, Nodes_count: 2418, Cost Time: 3.90s\n",
      "Time: 2019-05-11 01:06:24.691531973~2019-05-11 01:21:50.262931075, Loss: 0.6917, Nodes_count: 2726, Cost Time: 6.12s\n",
      "Time: 2019-05-11 01:21:50.262931075~2019-05-11 01:36:59.251529646, Loss: 0.2389, Nodes_count: 2989, Cost Time: 7.02s\n",
      "Time: 2019-05-11 01:36:59.251529646~2019-05-11 01:52:29.101240073, Loss: 0.1717, Nodes_count: 3054, Cost Time: 8.93s\n",
      "Time: 2019-05-11 01:52:29.101240073~2019-05-11 02:11:41.011246483, Loss: 0.6546, Nodes_count: 3087, Cost Time: 9.85s\n",
      "Time: 2019-05-11 02:11:41.011246483~2019-05-11 02:32:39.591557711, Loss: 0.4783, Nodes_count: 3204, Cost Time: 10.80s\n",
      "Time: 2019-05-11 02:32:39.591557711~2019-05-11 02:56:41.041596445, Loss: 0.4760, Nodes_count: 3291, Cost Time: 11.30s\n",
      "Time: 2019-05-11 02:56:41.041596445~2019-05-11 04:00:00.093595955, Loss: 1.3928, Nodes_count: 3401, Cost Time: 11.39s\n",
      "Time: 2019-05-11 04:00:00.093595955~2019-05-11 06:00:00.131044119, Loss: 0.7539, Nodes_count: 3404, Cost Time: 11.45s\n",
      "Time: 2019-05-11 06:00:00.131044119~2019-05-11 07:33:00.032549731, Loss: 1.1855, Nodes_count: 3404, Cost Time: 11.51s\n",
      "Time: 2019-05-11 07:33:00.032549731~2019-05-11 08:02:55.490748972, Loss: 2.5495, Nodes_count: 3549, Cost Time: 11.58s\n",
      "Time: 2019-05-11 08:02:55.490748972~2019-05-11 08:44:00.040714942, Loss: 1.5696, Nodes_count: 3673, Cost Time: 11.64s\n",
      "Time: 2019-05-11 08:44:00.040714942~2019-05-11 09:02:45.090734321, Loss: 0.8579, Nodes_count: 3675, Cost Time: 11.71s\n",
      "Time: 2019-05-11 09:02:45.090734321~2019-05-11 09:18:30.552522898, Loss: 0.5426, Nodes_count: 3822, Cost Time: 12.62s\n",
      "Time: 2019-05-11 09:18:30.552522898~2019-05-11 09:33:38.950817990, Loss: 0.2820, Nodes_count: 4048, Cost Time: 14.02s\n",
      "Time: 2019-05-11 09:33:38.950817990~2019-05-11 09:49:20.872291437, Loss: 0.4803, Nodes_count: 4213, Cost Time: 15.10s\n",
      "Time: 2019-05-11 09:49:20.872291437~2019-05-11 10:04:22.132286412, Loss: 0.3313, Nodes_count: 4439, Cost Time: 16.02s\n",
      "Time: 2019-05-11 10:04:22.132286412~2019-05-11 10:19:42.322328604, Loss: 0.3721, Nodes_count: 4711, Cost Time: 17.20s\n",
      "Time: 2019-05-11 10:19:42.322328604~2019-05-11 10:35:48.572174650, Loss: 0.5328, Nodes_count: 4913, Cost Time: 18.05s\n",
      "Time: 2019-05-11 10:35:48.572174650~2019-05-11 10:52:35.530558818, Loss: 0.3398, Nodes_count: 5160, Cost Time: 19.01s\n",
      "Time: 2019-05-11 10:52:35.530558818~2019-05-11 11:08:23.082538384, Loss: 0.3162, Nodes_count: 5355, Cost Time: 20.01s\n",
      "Time: 2019-05-11 11:08:23.082538384~2019-05-11 11:23:48.710934113, Loss: 0.3257, Nodes_count: 5620, Cost Time: 20.85s\n",
      "Time: 2019-05-11 11:23:48.710934113~2019-05-11 11:42:31.430627051, Loss: 0.4611, Nodes_count: 5923, Cost Time: 23.66s\n",
      "Time: 2019-05-11 11:42:31.430627051~2019-05-11 11:57:57.562022014, Loss: 0.5303, Nodes_count: 6058, Cost Time: 24.78s\n",
      "Time: 2019-05-11 11:57:57.562022014~2019-05-11 12:13:09.372352695, Loss: 0.4834, Nodes_count: 6215, Cost Time: 25.89s\n",
      "Time: 2019-05-11 12:13:09.372352695~2019-05-11 12:29:04.730612052, Loss: 0.2731, Nodes_count: 6431, Cost Time: 26.69s\n",
      "Time: 2019-05-11 12:29:04.730612052~2019-05-11 12:44:10.280376666, Loss: 0.4360, Nodes_count: 6704, Cost Time: 27.68s\n",
      "Time: 2019-05-11 12:44:10.280376666~2019-05-11 13:00:00.070653433, Loss: 0.3257, Nodes_count: 6808, Cost Time: 28.35s\n",
      "Time: 2019-05-11 13:00:00.070653433~2019-05-11 13:15:28.570369771, Loss: 0.4201, Nodes_count: 7051, Cost Time: 29.12s\n",
      "Time: 2019-05-11 13:15:28.570369771~2019-05-11 13:30:58.892272521, Loss: 0.4746, Nodes_count: 7164, Cost Time: 30.12s\n",
      "Time: 2019-05-11 13:30:58.892272521~2019-05-11 13:46:20.710386480, Loss: 0.2736, Nodes_count: 7447, Cost Time: 31.29s\n",
      "Time: 2019-05-11 13:46:20.710386480~2019-05-11 14:01:28.090361968, Loss: 0.3353, Nodes_count: 7636, Cost Time: 32.04s\n",
      "Time: 2019-05-11 14:01:28.090361968~2019-05-11 14:20:23.201867433, Loss: 0.5378, Nodes_count: 7741, Cost Time: 33.42s\n",
      "Time: 2019-05-11 14:20:23.201867433~2019-05-11 14:35:32.721889377, Loss: 0.3375, Nodes_count: 7916, Cost Time: 34.07s\n",
      "Time: 2019-05-11 14:35:32.721889377~2019-05-11 14:50:34.402040227, Loss: 0.5176, Nodes_count: 8042, Cost Time: 35.07s\n",
      "Time: 2019-05-11 14:50:34.402040227~2019-05-11 15:07:35.710277369, Loss: 0.5241, Nodes_count: 8187, Cost Time: 36.58s\n",
      "Time: 2019-05-11 15:07:35.710277369~2019-05-11 15:23:06.121816362, Loss: 0.6172, Nodes_count: 8514, Cost Time: 38.06s\n",
      "Time: 2019-05-11 15:23:06.121816362~2019-05-11 15:38:06.640111073, Loss: 0.4183, Nodes_count: 8593, Cost Time: 39.09s\n",
      "Time: 2019-05-11 15:38:06.640111073~2019-05-11 15:55:19.421701429, Loss: 0.2875, Nodes_count: 8742, Cost Time: 40.37s\n",
      "Time: 2019-05-11 15:55:19.421701429~2019-05-11 16:11:06.201712387, Loss: 0.1860, Nodes_count: 9005, Cost Time: 41.40s\n",
      "Time: 2019-05-11 16:11:06.201712387~2019-05-11 16:26:06.370413919, Loss: 0.3543, Nodes_count: 9193, Cost Time: 42.25s\n",
      "Time: 2019-05-11 16:26:06.370413919~2019-05-11 16:42:03.250211181, Loss: 0.4583, Nodes_count: 9302, Cost Time: 42.92s\n",
      "Time: 2019-05-11 16:42:03.250211181~2019-05-11 16:57:21.541759824, Loss: 0.3556, Nodes_count: 9491, Cost Time: 43.64s\n",
      "Time: 2019-05-11 16:57:21.541759824~2019-05-11 17:13:50.030194213, Loss: 0.4180, Nodes_count: 9755, Cost Time: 44.81s\n",
      "Time: 2019-05-11 17:13:50.030194213~2019-05-11 17:29:35.611633615, Loss: 0.3767, Nodes_count: 9935, Cost Time: 45.77s\n",
      "Time: 2019-05-11 17:29:35.611633615~2019-05-11 17:44:41.101731269, Loss: 0.5915, Nodes_count: 10031, Cost Time: 46.68s\n",
      "Time: 2019-05-11 17:44:41.101731269~2019-05-11 17:59:48.200011701, Loss: 0.4865, Nodes_count: 10108, Cost Time: 47.96s\n",
      "Time: 2019-05-11 17:59:48.200011701~2019-05-11 18:14:58.070201896, Loss: 0.2682, Nodes_count: 10341, Cost Time: 48.93s\n",
      "Time: 2019-05-11 18:14:58.070201896~2019-05-11 18:31:24.531852672, Loss: 0.6500, Nodes_count: 10471, Cost Time: 52.02s\n",
      "Time: 2019-05-11 18:31:24.531852672~2019-05-11 18:46:30.431486539, Loss: 0.2804, Nodes_count: 10705, Cost Time: 54.05s\n",
      "Time: 2019-05-11 18:46:30.431486539~2019-05-11 19:04:49.451463170, Loss: 0.3449, Nodes_count: 10787, Cost Time: 54.98s\n",
      "Time: 2019-05-11 19:04:49.451463170~2019-05-11 19:20:45.001751611, Loss: 0.7615, Nodes_count: 10991, Cost Time: 56.54s\n",
      "Time: 2019-05-11 19:20:45.001751611~2019-05-11 19:35:59.261743466, Loss: 0.5358, Nodes_count: 11206, Cost Time: 57.53s\n",
      "Time: 2019-05-11 19:35:59.261743466~2019-05-11 19:51:41.109819328, Loss: 0.6194, Nodes_count: 11294, Cost Time: 58.15s\n",
      "Time: 2019-05-11 19:51:41.109819328~2019-05-11 20:08:09.601578521, Loss: 0.4534, Nodes_count: 11643, Cost Time: 60.30s\n",
      "Time: 2019-05-11 20:08:09.601578521~2019-05-11 20:24:50.761409597, Loss: 0.4116, Nodes_count: 11790, Cost Time: 61.20s\n",
      "Time: 2019-05-11 20:24:50.761409597~2019-05-11 20:41:19.861366854, Loss: 0.5448, Nodes_count: 12015, Cost Time: 62.48s\n",
      "Time: 2019-05-11 20:41:19.861366854~2019-05-11 20:56:21.361466559, Loss: 0.4826, Nodes_count: 12318, Cost Time: 63.78s\n",
      "Time: 2019-05-11 20:56:21.361466559~2019-05-11 21:11:51.881465525, Loss: 0.4735, Nodes_count: 12434, Cost Time: 64.37s\n",
      "Time: 2019-05-11 21:11:51.881465525~2019-05-11 21:27:12.179927255, Loss: 0.6138, Nodes_count: 12568, Cost Time: 65.02s\n",
      "Time: 2019-05-11 21:27:12.179927255~2019-05-11 21:43:46.639964133, Loss: 0.4111, Nodes_count: 12699, Cost Time: 65.75s\n",
      "Time: 2019-05-11 21:43:46.639964133~2019-05-11 22:00:35.469784723, Loss: 0.4960, Nodes_count: 12913, Cost Time: 66.91s\n",
      "Time: 2019-05-11 22:00:35.469784723~2019-05-11 22:15:37.111489858, Loss: 0.4978, Nodes_count: 12995, Cost Time: 67.83s\n",
      "Time: 2019-05-11 22:15:37.111489858~2019-05-11 22:31:41.441418988, Loss: 0.3396, Nodes_count: 13070, Cost Time: 68.82s\n",
      "Time: 2019-05-11 22:31:41.441418988~2019-05-11 22:47:17.341548629, Loss: 0.3541, Nodes_count: 13316, Cost Time: 69.84s\n",
      "Time: 2019-05-11 22:47:17.341548629~2019-05-11 23:02:31.751582100, Loss: 0.4884, Nodes_count: 13461, Cost Time: 70.86s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time: 2019-05-11 23:02:31.751582100~2019-05-11 23:17:50.041154910, Loss: 0.4764, Nodes_count: 14801, Cost Time: 72.69s\n",
      "Time: 2019-05-11 23:17:50.041154910~2019-05-11 23:32:58.392191613, Loss: 0.3470, Nodes_count: 15078, Cost Time: 73.99s\n",
      "Time: 2019-05-11 23:32:58.392191613~2019-05-11 23:49:00.841202714, Loss: 0.6267, Nodes_count: 15232, Cost Time: 74.96s\n"
     ]
    }
   ],
   "source": [
    "ans_5_11=test_day_new(graph_5_11,\"./graph_5_11\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after merge: TemporalData(dst=[1404800], msg=[1404800, 41], src=[1404800], t=[1404800])\n",
      "Time: 2019-05-12 00:00:00.009704889~2019-05-12 00:15:44.829441975, Loss: 0.6369, Nodes_count: 850, Cost Time: 1.56s\n",
      "Time: 2019-05-12 00:15:44.829441975~2019-05-12 00:31:07.179657269, Loss: 0.5119, Nodes_count: 1315, Cost Time: 2.43s\n",
      "Time: 2019-05-12 00:31:07.179657269~2019-05-12 00:46:11.631021123, Loss: 0.7138, Nodes_count: 1645, Cost Time: 5.02s\n",
      "Time: 2019-05-12 00:46:11.631021123~2019-05-12 01:01:22.279490585, Loss: 0.3149, Nodes_count: 2095, Cost Time: 6.27s\n",
      "Time: 2019-05-12 01:01:22.279490585~2019-05-12 01:16:57.199370459, Loss: 0.6949, Nodes_count: 2393, Cost Time: 7.72s\n",
      "Time: 2019-05-12 01:16:57.199370459~2019-05-12 01:34:43.379773300, Loss: 0.3989, Nodes_count: 2531, Cost Time: 8.25s\n",
      "Time: 2019-05-12 01:34:43.379773300~2019-05-12 01:50:01.881070816, Loss: 0.5140, Nodes_count: 2882, Cost Time: 9.59s\n",
      "Time: 2019-05-12 01:50:01.881070816~2019-05-12 02:05:21.991043789, Loss: 0.5319, Nodes_count: 3100, Cost Time: 10.50s\n",
      "Time: 2019-05-12 02:05:21.991043789~2019-05-12 02:28:34.460864379, Loss: 0.5278, Nodes_count: 3226, Cost Time: 11.33s\n",
      "Time: 2019-05-12 02:28:34.460864379~2019-05-12 02:43:35.679241588, Loss: 0.4144, Nodes_count: 3497, Cost Time: 12.11s\n",
      "Time: 2019-05-12 02:43:35.679241588~2019-05-12 03:00:21.411945897, Loss: 0.7205, Nodes_count: 3785, Cost Time: 14.70s\n",
      "Time: 2019-05-12 03:00:21.411945897~2019-05-12 03:19:12.990795964, Loss: 0.4323, Nodes_count: 3985, Cost Time: 17.20s\n",
      "Time: 2019-05-12 03:19:12.990795964~2019-05-12 03:35:04.580872340, Loss: 0.5443, Nodes_count: 4078, Cost Time: 18.06s\n",
      "Time: 2019-05-12 03:35:04.580872340~2019-05-12 03:51:00.770915470, Loss: 0.4823, Nodes_count: 4230, Cost Time: 18.61s\n",
      "Time: 2019-05-12 03:51:00.770915470~2019-05-12 04:14:25.980718135, Loss: 0.4996, Nodes_count: 4261, Cost Time: 18.87s\n",
      "Time: 2019-05-12 04:14:25.980718135~2019-05-12 04:30:03.821047974, Loss: 0.2415, Nodes_count: 4294, Cost Time: 19.17s\n",
      "Time: 2019-05-12 04:30:03.821047974~2019-05-12 04:52:44.540757307, Loss: 0.4330, Nodes_count: 4349, Cost Time: 19.42s\n",
      "Time: 2019-05-12 04:52:44.540757307~2019-05-12 06:11:00.031542504, Loss: 0.4492, Nodes_count: 4354, Cost Time: 19.49s\n",
      "Time: 2019-05-12 06:11:00.031542504~2019-05-12 08:19:26.020434833, Loss: 0.5402, Nodes_count: 4359, Cost Time: 19.55s\n",
      "Time: 2019-05-12 08:19:26.020434833~2019-05-12 09:02:13.260371924, Loss: 1.2753, Nodes_count: 4383, Cost Time: 19.61s\n",
      "Time: 2019-05-12 09:02:13.260371924~2019-05-12 09:17:48.320379390, Loss: 0.3966, Nodes_count: 4497, Cost Time: 20.18s\n",
      "Time: 2019-05-12 09:17:48.320379390~2019-05-12 09:33:06.770434518, Loss: 0.5655, Nodes_count: 4626, Cost Time: 21.34s\n",
      "Time: 2019-05-12 09:33:06.770434518~2019-05-12 09:48:33.950339246, Loss: 0.4517, Nodes_count: 4753, Cost Time: 22.08s\n",
      "Time: 2019-05-12 09:48:33.950339246~2019-05-12 10:03:59.991169677, Loss: 0.4349, Nodes_count: 5065, Cost Time: 23.58s\n",
      "Time: 2019-05-12 10:03:59.991169677~2019-05-12 10:19:13.590312580, Loss: 0.5431, Nodes_count: 5150, Cost Time: 24.67s\n",
      "Time: 2019-05-12 10:19:13.590312580~2019-05-12 10:35:42.188942646, Loss: 0.5237, Nodes_count: 5451, Cost Time: 25.30s\n",
      "Time: 2019-05-12 10:35:42.188942646~2019-05-12 10:50:59.070373959, Loss: 0.5453, Nodes_count: 5590, Cost Time: 26.29s\n",
      "Time: 2019-05-12 10:50:59.070373959~2019-05-12 11:06:55.398596279, Loss: 0.5612, Nodes_count: 5710, Cost Time: 26.80s\n",
      "Time: 2019-05-12 11:06:55.398596279~2019-05-12 11:22:19.110356388, Loss: 0.4551, Nodes_count: 5938, Cost Time: 27.95s\n",
      "Time: 2019-05-12 11:22:19.110356388~2019-05-12 11:39:14.648548872, Loss: 0.4907, Nodes_count: 6046, Cost Time: 28.80s\n",
      "Time: 2019-05-12 11:39:14.648548872~2019-05-12 11:54:48.270493584, Loss: 0.4472, Nodes_count: 6263, Cost Time: 29.92s\n",
      "Time: 2019-05-12 11:54:48.270493584~2019-05-12 12:11:11.058470422, Loss: 0.3944, Nodes_count: 6395, Cost Time: 30.60s\n",
      "Time: 2019-05-12 12:11:11.058470422~2019-05-12 12:28:07.050478081, Loss: 0.4147, Nodes_count: 6501, Cost Time: 31.21s\n",
      "Time: 2019-05-12 12:28:07.050478081~2019-05-12 12:45:55.988545214, Loss: 0.5515, Nodes_count: 6617, Cost Time: 32.03s\n",
      "Time: 2019-05-12 12:45:55.988545214~2019-05-12 13:01:25.358430399, Loss: 0.4660, Nodes_count: 6861, Cost Time: 32.87s\n",
      "Time: 2019-05-12 13:01:25.358430399~2019-05-12 13:16:51.510105165, Loss: 0.4589, Nodes_count: 7064, Cost Time: 33.83s\n",
      "Time: 2019-05-12 13:16:51.510105165~2019-05-12 13:32:11.088567055, Loss: 0.5393, Nodes_count: 7119, Cost Time: 34.33s\n",
      "Time: 2019-05-12 13:32:11.088567055~2019-05-12 13:49:26.740053106, Loss: 0.4293, Nodes_count: 7225, Cost Time: 34.92s\n",
      "Time: 2019-05-12 13:49:26.740053106~2019-05-12 14:04:39.348401026, Loss: 0.5034, Nodes_count: 7282, Cost Time: 35.69s\n",
      "Time: 2019-05-12 14:04:39.348401026~2019-05-12 14:20:00.718474822, Loss: 0.4224, Nodes_count: 7412, Cost Time: 36.80s\n",
      "Time: 2019-05-12 14:20:00.718474822~2019-05-12 14:37:14.118293891, Loss: 0.4759, Nodes_count: 7558, Cost Time: 37.93s\n",
      "Time: 2019-05-12 14:37:14.118293891~2019-05-12 14:56:01.138390972, Loss: 0.4243, Nodes_count: 7870, Cost Time: 39.35s\n",
      "Time: 2019-05-12 14:56:01.138390972~2019-05-12 15:14:15.209887927, Loss: 0.6754, Nodes_count: 7953, Cost Time: 39.83s\n",
      "Time: 2019-05-12 15:14:15.209887927~2019-05-12 15:29:30.608227334, Loss: 0.4024, Nodes_count: 8127, Cost Time: 40.89s\n",
      "Time: 2019-05-12 15:29:30.608227334~2019-05-12 15:44:37.278232613, Loss: 0.2704, Nodes_count: 8434, Cost Time: 43.04s\n",
      "Time: 2019-05-12 15:44:37.278232613~2019-05-12 16:01:50.240056285, Loss: 0.5151, Nodes_count: 8588, Cost Time: 43.99s\n",
      "Time: 2019-05-12 16:01:50.240056285~2019-05-12 16:17:34.358244357, Loss: 0.3638, Nodes_count: 8686, Cost Time: 44.77s\n",
      "Time: 2019-05-12 16:17:34.358244357~2019-05-12 16:32:49.778405684, Loss: 0.4538, Nodes_count: 8869, Cost Time: 45.50s\n",
      "Time: 2019-05-12 16:32:49.778405684~2019-05-12 16:49:35.808279210, Loss: 0.3476, Nodes_count: 9156, Cost Time: 46.61s\n",
      "Time: 2019-05-12 16:49:35.808279210~2019-05-12 17:05:32.249713239, Loss: 0.2839, Nodes_count: 9297, Cost Time: 47.40s\n",
      "Time: 2019-05-12 17:05:32.249713239~2019-05-12 17:24:35.038337089, Loss: 0.3083, Nodes_count: 9626, Cost Time: 48.73s\n",
      "Time: 2019-05-12 17:24:35.038337089~2019-05-12 17:41:10.199756733, Loss: 0.5188, Nodes_count: 9688, Cost Time: 49.42s\n",
      "Time: 2019-05-12 17:41:10.199756733~2019-05-12 17:59:01.508131935, Loss: 0.4992, Nodes_count: 9847, Cost Time: 50.75s\n",
      "Time: 2019-05-12 17:59:01.508131935~2019-05-12 18:14:03.449724644, Loss: 0.4320, Nodes_count: 9977, Cost Time: 51.49s\n",
      "Time: 2019-05-12 18:14:03.449724644~2019-05-12 18:29:06.678076897, Loss: 0.5584, Nodes_count: 10163, Cost Time: 53.97s\n",
      "Time: 2019-05-12 18:29:06.678076897~2019-05-12 18:44:27.669830882, Loss: 0.3761, Nodes_count: 10234, Cost Time: 55.58s\n",
      "Time: 2019-05-12 18:44:27.669830882~2019-05-12 18:59:28.338254204, Loss: 0.4425, Nodes_count: 10328, Cost Time: 56.29s\n",
      "Time: 2019-05-12 18:59:28.338254204~2019-05-12 19:15:25.609609078, Loss: 0.6144, Nodes_count: 10428, Cost Time: 57.07s\n",
      "Time: 2019-05-12 19:15:25.609609078~2019-05-12 19:31:00.599805001, Loss: 0.5448, Nodes_count: 10625, Cost Time: 58.63s\n",
      "Time: 2019-05-12 19:31:00.599805001~2019-05-12 19:47:24.227961481, Loss: 0.5037, Nodes_count: 10787, Cost Time: 59.68s\n",
      "Time: 2019-05-12 19:47:24.227961481~2019-05-12 20:02:25.699599248, Loss: 0.4231, Nodes_count: 11001, Cost Time: 61.56s\n",
      "Time: 2019-05-12 20:02:25.699599248~2019-05-12 20:18:42.717871146, Loss: 0.7637, Nodes_count: 11159, Cost Time: 63.20s\n",
      "Time: 2019-05-12 20:18:42.717871146~2019-05-12 20:35:40.469447008, Loss: 0.4324, Nodes_count: 11277, Cost Time: 64.02s\n",
      "Time: 2019-05-12 20:35:40.469447008~2019-05-12 20:53:05.528165477, Loss: 0.6483, Nodes_count: 11420, Cost Time: 65.40s\n",
      "Time: 2019-05-12 20:53:05.528165477~2019-05-12 21:09:28.789735522, Loss: 0.4422, Nodes_count: 11637, Cost Time: 66.38s\n",
      "Time: 2019-05-12 21:09:28.789735522~2019-05-12 21:26:57.479438103, Loss: 0.6077, Nodes_count: 11678, Cost Time: 67.10s\n",
      "Time: 2019-05-12 21:26:57.479438103~2019-05-12 21:41:59.739502211, Loss: 0.6120, Nodes_count: 11774, Cost Time: 67.88s\n",
      "Time: 2019-05-12 21:41:59.739502211~2019-05-12 21:57:12.129756843, Loss: 0.4083, Nodes_count: 12023, Cost Time: 68.95s\n",
      "Time: 2019-05-12 21:57:12.129756843~2019-05-12 22:15:24.049648957, Loss: 0.3846, Nodes_count: 12282, Cost Time: 70.21s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time: 2019-05-12 22:15:24.049648957~2019-05-12 22:30:34.307913598, Loss: 0.3599, Nodes_count: 12539, Cost Time: 72.21s\n",
      "Time: 2019-05-12 22:30:34.307913598~2019-05-12 22:45:52.147868084, Loss: 0.2700, Nodes_count: 12824, Cost Time: 73.15s\n",
      "Time: 2019-05-12 22:45:52.147868084~2019-05-12 23:01:24.217632826, Loss: 0.6649, Nodes_count: 13009, Cost Time: 75.14s\n",
      "Time: 2019-05-12 23:01:24.217632826~2019-05-12 23:16:34.389656674, Loss: 0.5139, Nodes_count: 13131, Cost Time: 76.80s\n",
      "Time: 2019-05-12 23:16:34.389656674~2019-05-12 23:33:07.037830799, Loss: 0.6692, Nodes_count: 13840, Cost Time: 78.47s\n",
      "Time: 2019-05-12 23:33:07.037830799~2019-05-12 23:48:57.527662974, Loss: 0.5240, Nodes_count: 14683, Cost Time: 80.10s\n"
     ]
    }
   ],
   "source": [
    "ans_5_12=test_day_new(graph_5_12,\"./graph_5_12\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after merge: TemporalData(dst=[5841235], msg=[5841235, 41], src=[5841235], t=[5841235])\n",
      "Time: 2019-05-15 00:00:00.006192320~2019-05-15 00:15:13.743767966, Loss: 0.4844, Nodes_count: 1709, Cost Time: 5.28s\n",
      "Time: 2019-05-15 00:15:13.743767966~2019-05-15 00:30:15.346246304, Loss: 0.4370, Nodes_count: 2252, Cost Time: 8.96s\n",
      "Time: 2019-05-15 00:30:15.346246304~2019-05-15 00:45:38.935375235, Loss: 0.4267, Nodes_count: 2810, Cost Time: 15.06s\n",
      "Time: 2019-05-15 00:45:38.935375235~2019-05-15 01:00:49.986348448, Loss: 0.3025, Nodes_count: 3070, Cost Time: 18.67s\n",
      "Time: 2019-05-15 01:00:49.986348448~2019-05-15 01:16:13.516335500, Loss: 0.3608, Nodes_count: 3402, Cost Time: 24.38s\n",
      "Time: 2019-05-15 01:16:13.516335500~2019-05-15 01:31:15.876102508, Loss: 0.3794, Nodes_count: 3734, Cost Time: 30.00s\n",
      "Time: 2019-05-15 01:31:15.876102508~2019-05-15 01:46:35.756168353, Loss: 0.3168, Nodes_count: 3885, Cost Time: 33.51s\n",
      "Time: 2019-05-15 01:46:35.756168353~2019-05-15 02:02:03.445171413, Loss: 0.5027, Nodes_count: 4075, Cost Time: 36.45s\n",
      "Time: 2019-05-15 02:02:03.445171413~2019-05-15 02:17:36.366097118, Loss: 0.5171, Nodes_count: 4426, Cost Time: 42.46s\n",
      "Time: 2019-05-15 02:17:36.366097118~2019-05-15 02:33:08.316023927, Loss: 0.5576, Nodes_count: 4573, Cost Time: 44.31s\n",
      "Time: 2019-05-15 02:33:08.316023927~2019-05-15 02:48:12.346047654, Loss: 0.2702, Nodes_count: 4752, Cost Time: 49.18s\n",
      "Time: 2019-05-15 02:48:12.346047654~2019-05-15 03:04:19.506191624, Loss: 0.3904, Nodes_count: 4911, Cost Time: 51.09s\n",
      "Time: 2019-05-15 03:04:19.506191624~2019-05-15 03:19:20.336382785, Loss: 0.3315, Nodes_count: 5134, Cost Time: 54.93s\n",
      "Time: 2019-05-15 03:19:20.336382785~2019-05-15 03:34:24.565949954, Loss: 0.2550, Nodes_count: 5249, Cost Time: 58.31s\n",
      "Time: 2019-05-15 03:34:24.565949954~2019-05-15 03:50:08.335957105, Loss: 0.1692, Nodes_count: 5250, Cost Time: 59.47s\n",
      "Time: 2019-05-15 03:50:08.335957105~2019-05-15 04:07:57.715926628, Loss: 0.0423, Nodes_count: 5250, Cost Time: 59.86s\n",
      "Time: 2019-05-15 04:07:57.715926628~2019-05-15 04:25:32.335891238, Loss: 0.0378, Nodes_count: 5250, Cost Time: 60.21s\n",
      "Time: 2019-05-15 04:25:32.335891238~2019-05-15 04:41:16.315832364, Loss: 0.0377, Nodes_count: 5250, Cost Time: 60.51s\n",
      "Time: 2019-05-15 04:41:16.315832364~2019-05-15 04:56:36.326111458, Loss: 0.0337, Nodes_count: 5251, Cost Time: 60.80s\n",
      "Time: 2019-05-15 04:56:36.326111458~2019-05-15 05:13:08.315787838, Loss: 0.0343, Nodes_count: 5253, Cost Time: 61.15s\n",
      "Time: 2019-05-15 05:13:08.315787838~2019-05-15 05:28:36.315778004, Loss: 0.0340, Nodes_count: 5253, Cost Time: 61.45s\n",
      "Time: 2019-05-15 05:28:36.315778004~2019-05-15 05:44:40.326005052, Loss: 0.0292, Nodes_count: 5253, Cost Time: 61.74s\n",
      "Time: 2019-05-15 05:44:40.326005052~2019-05-15 06:00:28.326014326, Loss: 0.0729, Nodes_count: 5281, Cost Time: 62.04s\n",
      "Time: 2019-05-15 06:00:28.326014326~2019-05-15 06:17:32.325991084, Loss: 0.0357, Nodes_count: 5281, Cost Time: 62.39s\n",
      "Time: 2019-05-15 06:17:32.325991084~2019-05-15 06:33:44.315665234, Loss: 0.0308, Nodes_count: 5281, Cost Time: 62.69s\n",
      "Time: 2019-05-15 06:33:44.315665234~2019-05-15 06:49:36.336078067, Loss: 0.0402, Nodes_count: 5281, Cost Time: 62.98s\n",
      "Time: 2019-05-15 06:49:36.336078067~2019-05-15 07:05:22.335601187, Loss: 0.0436, Nodes_count: 5281, Cost Time: 63.28s\n",
      "Time: 2019-05-15 07:05:22.335601187~2019-05-15 07:23:00.336037899, Loss: 0.0370, Nodes_count: 5281, Cost Time: 63.63s\n",
      "Time: 2019-05-15 07:23:00.336037899~2019-05-15 07:39:16.336018368, Loss: 0.0380, Nodes_count: 5281, Cost Time: 63.93s\n",
      "Time: 2019-05-15 07:39:16.336018368~2019-05-15 07:55:36.325862117, Loss: 0.0353, Nodes_count: 5281, Cost Time: 64.22s\n",
      "Time: 2019-05-15 07:55:36.325862117~2019-05-15 08:12:44.335538340, Loss: 0.0338, Nodes_count: 5281, Cost Time: 64.57s\n",
      "Time: 2019-05-15 08:12:44.335538340~2019-05-15 08:29:08.335580692, Loss: 0.0296, Nodes_count: 5281, Cost Time: 64.87s\n",
      "Time: 2019-05-15 08:29:08.335580692~2019-05-15 08:46:56.325786951, Loss: 0.3628, Nodes_count: 5300, Cost Time: 65.22s\n",
      "Time: 2019-05-15 08:46:56.325786951~2019-05-15 09:02:05.234725465, Loss: 0.3407, Nodes_count: 5361, Cost Time: 65.63s\n",
      "Time: 2019-05-15 09:02:05.234725465~2019-05-15 09:17:07.225888229, Loss: 0.3537, Nodes_count: 5721, Cost Time: 71.19s\n",
      "Time: 2019-05-15 09:17:07.225888229~2019-05-15 09:32:11.065706817, Loss: 0.3489, Nodes_count: 5863, Cost Time: 74.63s\n",
      "Time: 2019-05-15 09:32:11.065706817~2019-05-15 09:48:38.684621579, Loss: 0.4089, Nodes_count: 5976, Cost Time: 80.09s\n",
      "Time: 2019-05-15 09:48:38.684621579~2019-05-15 10:03:41.465554650, Loss: 0.6036, Nodes_count: 6154, Cost Time: 83.91s\n",
      "Time: 2019-05-15 10:03:41.465554650~2019-05-15 10:18:48.315349922, Loss: 0.3026, Nodes_count: 6368, Cost Time: 88.02s\n",
      "Time: 2019-05-15 10:18:48.315349922~2019-05-15 10:34:18.902982618, Loss: 0.5541, Nodes_count: 6581, Cost Time: 92.47s\n",
      "Time: 2019-05-15 10:34:18.902982618~2019-05-15 10:49:19.662777306, Loss: 0.4913, Nodes_count: 6711, Cost Time: 95.75s\n",
      "Time: 2019-05-15 10:49:19.662777306~2019-05-15 11:04:34.935316812, Loss: 0.5303, Nodes_count: 6841, Cost Time: 101.36s\n",
      "Time: 2019-05-15 11:04:34.935316812~2019-05-15 11:20:14.345345440, Loss: 0.4619, Nodes_count: 7045, Cost Time: 105.57s\n",
      "Time: 2019-05-15 11:20:14.345345440~2019-05-15 11:35:14.525271329, Loss: 0.4388, Nodes_count: 7263, Cost Time: 112.02s\n",
      "Time: 2019-05-15 11:35:14.525271329~2019-05-15 11:51:01.435516598, Loss: 0.3626, Nodes_count: 7512, Cost Time: 117.25s\n",
      "Time: 2019-05-15 11:51:01.435516598~2019-05-15 12:07:06.345256228, Loss: 0.6116, Nodes_count: 7804, Cost Time: 123.21s\n",
      "Time: 2019-05-15 12:07:06.345256228~2019-05-15 12:22:23.514541754, Loss: 0.4335, Nodes_count: 8131, Cost Time: 130.08s\n",
      "Time: 2019-05-15 12:22:23.514541754~2019-05-15 12:38:28.335591405, Loss: 0.3119, Nodes_count: 8259, Cost Time: 135.25s\n",
      "Time: 2019-05-15 12:38:28.335591405~2019-05-15 12:53:47.114274288, Loss: 0.4252, Nodes_count: 8351, Cost Time: 138.42s\n",
      "Time: 2019-05-15 12:53:47.114274288~2019-05-15 13:08:48.445537909, Loss: 0.7790, Nodes_count: 8773, Cost Time: 146.25s\n",
      "Time: 2019-05-15 13:08:48.445537909~2019-05-15 13:23:52.255450878, Loss: 0.4234, Nodes_count: 8967, Cost Time: 150.01s\n",
      "Time: 2019-05-15 13:23:52.255450878~2019-05-15 13:39:04.835250858, Loss: 0.5990, Nodes_count: 9246, Cost Time: 153.73s\n",
      "Time: 2019-05-15 13:39:04.835250858~2019-05-15 13:54:20.623015847, Loss: 0.2983, Nodes_count: 9459, Cost Time: 158.88s\n",
      "Time: 2019-05-15 13:54:20.623015847~2019-05-15 14:09:38.985029953, Loss: 0.4421, Nodes_count: 9646, Cost Time: 163.11s\n",
      "Time: 2019-05-15 14:09:38.985029953~2019-05-15 14:25:00.732554067, Loss: 0.4228, Nodes_count: 9903, Cost Time: 167.63s\n",
      "Time: 2019-05-15 14:25:00.732554067~2019-05-15 14:40:58.314993755, Loss: 0.4786, Nodes_count: 10105, Cost Time: 171.32s\n",
      "Time: 2019-05-15 14:40:58.314993755~2019-05-15 14:56:22.325270164, Loss: 0.6370, Nodes_count: 10343, Cost Time: 177.34s\n",
      "Time: 2019-05-15 14:56:22.325270164~2019-05-15 15:11:29.355153391, Loss: 0.4599, Nodes_count: 10578, Cost Time: 183.41s\n",
      "Time: 2019-05-15 15:11:29.355153391~2019-05-15 15:26:30.315069987, Loss: 0.5815, Nodes_count: 10858, Cost Time: 189.06s\n",
      "Time: 2019-05-15 15:26:30.315069987~2019-05-15 15:41:58.325198150, Loss: 0.5206, Nodes_count: 11046, Cost Time: 191.33s\n",
      "Time: 2019-05-15 15:41:58.325198150~2019-05-15 15:56:59.265333194, Loss: 0.2139, Nodes_count: 11212, Cost Time: 193.59s\n",
      "Time: 2019-05-15 15:56:59.265333194~2019-05-15 16:12:04.095060867, Loss: 0.3137, Nodes_count: 11391, Cost Time: 197.13s\n",
      "Time: 2019-05-15 16:12:04.095060867~2019-05-15 16:27:36.075256581, Loss: 0.2894, Nodes_count: 11698, Cost Time: 202.72s\n",
      "Time: 2019-05-15 16:27:36.075256581~2019-05-15 16:43:24.282499513, Loss: 0.5258, Nodes_count: 12112, Cost Time: 206.43s\n",
      "Time: 2019-05-15 16:43:24.282499513~2019-05-15 16:58:34.225243276, Loss: 0.5314, Nodes_count: 12246, Cost Time: 208.17s\n",
      "Time: 2019-05-15 16:58:34.225243276~2019-05-15 17:13:45.412245202, Loss: 0.5421, Nodes_count: 12441, Cost Time: 211.77s\n",
      "Time: 2019-05-15 17:13:45.412245202~2019-05-15 17:28:46.064963493, Loss: 0.4824, Nodes_count: 12548, Cost Time: 214.78s\n",
      "Time: 2019-05-15 17:28:46.064963493~2019-05-15 17:44:50.054854387, Loss: 0.5381, Nodes_count: 12794, Cost Time: 219.85s\n",
      "Time: 2019-05-15 17:44:50.054854387~2019-05-15 17:59:50.764934674, Loss: 0.5492, Nodes_count: 12922, Cost Time: 222.70s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time: 2019-05-15 17:59:50.764934674~2019-05-15 18:15:04.084749724, Loss: 0.5200, Nodes_count: 13181, Cost Time: 226.33s\n",
      "Time: 2019-05-15 18:15:04.084749724~2019-05-15 18:30:12.692563554, Loss: 0.5208, Nodes_count: 13330, Cost Time: 229.22s\n",
      "Time: 2019-05-15 18:30:12.692563554~2019-05-15 18:45:36.312255908, Loss: 0.4195, Nodes_count: 13647, Cost Time: 234.54s\n",
      "Time: 2019-05-15 18:45:36.312255908~2019-05-15 19:00:43.932284310, Loss: 0.5164, Nodes_count: 13820, Cost Time: 239.27s\n",
      "Time: 2019-05-15 19:00:43.932284310~2019-05-15 19:16:40.834769346, Loss: 0.4172, Nodes_count: 14111, Cost Time: 244.85s\n",
      "Time: 2019-05-15 19:16:40.834769346~2019-05-15 19:32:22.054614132, Loss: 0.4489, Nodes_count: 14550, Cost Time: 251.23s\n",
      "Time: 2019-05-15 19:32:22.054614132~2019-05-15 19:47:26.074698234, Loss: 0.6650, Nodes_count: 14698, Cost Time: 255.42s\n",
      "Time: 2019-05-15 19:47:26.074698234~2019-05-15 20:02:51.452177789, Loss: 0.5192, Nodes_count: 14923, Cost Time: 259.31s\n",
      "Time: 2019-05-15 20:02:51.452177789~2019-05-15 20:18:20.054640718, Loss: 0.4605, Nodes_count: 15008, Cost Time: 261.22s\n",
      "Time: 2019-05-15 20:18:20.054640718~2019-05-15 20:33:20.173943586, Loss: 0.3281, Nodes_count: 15227, Cost Time: 265.40s\n",
      "Time: 2019-05-15 20:33:20.173943586~2019-05-15 20:48:26.344519703, Loss: 0.7387, Nodes_count: 15411, Cost Time: 269.26s\n",
      "Time: 2019-05-15 20:48:26.344519703~2019-05-15 21:03:34.932076311, Loss: 0.4981, Nodes_count: 15825, Cost Time: 274.58s\n",
      "Time: 2019-05-15 21:03:34.932076311~2019-05-15 21:18:36.594847325, Loss: 0.4275, Nodes_count: 16023, Cost Time: 278.87s\n",
      "Time: 2019-05-15 21:18:36.594847325~2019-05-15 21:33:37.744873139, Loss: 0.3652, Nodes_count: 16312, Cost Time: 282.72s\n",
      "Time: 2019-05-15 21:33:37.744873139~2019-05-15 21:48:38.194423026, Loss: 0.5610, Nodes_count: 16618, Cost Time: 289.57s\n",
      "Time: 2019-05-15 21:48:38.194423026~2019-05-15 22:03:43.404738808, Loss: 0.4743, Nodes_count: 16830, Cost Time: 294.75s\n",
      "Time: 2019-05-15 22:03:43.404738808~2019-05-15 22:19:00.063809691, Loss: 0.6533, Nodes_count: 17123, Cost Time: 298.15s\n",
      "Time: 2019-05-15 22:19:00.063809691~2019-05-15 22:34:04.714368495, Loss: 0.4169, Nodes_count: 17436, Cost Time: 303.69s\n",
      "Time: 2019-05-15 22:34:04.714368495~2019-05-15 22:49:14.843539280, Loss: 0.4182, Nodes_count: 17502, Cost Time: 306.94s\n",
      "Time: 2019-05-15 22:49:14.843539280~2019-05-15 23:04:18.584315206, Loss: 0.6606, Nodes_count: 17661, Cost Time: 310.75s\n",
      "Time: 2019-05-15 23:04:18.584315206~2019-05-15 23:19:21.464426330, Loss: 0.4760, Nodes_count: 17824, Cost Time: 314.37s\n",
      "Time: 2019-05-15 23:19:21.464426330~2019-05-15 23:34:24.683386847, Loss: 0.4938, Nodes_count: 18891, Cost Time: 320.89s\n",
      "Time: 2019-05-15 23:34:24.683386847~2019-05-15 23:49:33.091873325, Loss: 0.3430, Nodes_count: 19261, Cost Time: 326.29s\n"
     ]
    }
   ],
   "source": [
    "ans_5_15=test_day_new(graph_5_15,\"./graph_5_15\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after merge: TemporalData(dst=[5285414], msg=[5285414, 41], src=[5285414], t=[5285414])\n",
      "Time: 2019-05-16 00:00:00.014304910~2019-05-16 00:15:02.554575844, Loss: 0.6715, Nodes_count: 804, Cost Time: 2.05s\n",
      "Time: 2019-05-16 00:15:02.554575844~2019-05-16 00:30:19.324454842, Loss: 0.4468, Nodes_count: 1482, Cost Time: 5.96s\n",
      "Time: 2019-05-16 00:30:19.324454842~2019-05-16 00:45:31.114609140, Loss: 0.5222, Nodes_count: 1795, Cost Time: 9.19s\n",
      "Time: 2019-05-16 00:45:31.114609140~2019-05-16 01:01:06.524611321, Loss: 0.3684, Nodes_count: 2285, Cost Time: 14.90s\n",
      "Time: 2019-05-16 01:01:06.524611321~2019-05-16 01:17:54.074506572, Loss: 0.3526, Nodes_count: 2736, Cost Time: 17.08s\n",
      "Time: 2019-05-16 01:17:54.074506572~2019-05-16 01:32:54.901798763, Loss: 0.3481, Nodes_count: 3005, Cost Time: 18.41s\n",
      "Time: 2019-05-16 01:32:54.901798763~2019-05-16 01:48:08.801926929, Loss: 0.4402, Nodes_count: 3422, Cost Time: 20.29s\n",
      "Time: 2019-05-16 01:48:08.801926929~2019-05-16 02:03:13.643316639, Loss: 0.4401, Nodes_count: 3559, Cost Time: 21.45s\n",
      "Time: 2019-05-16 02:03:13.643316639~2019-05-16 02:19:39.961526318, Loss: 0.4134, Nodes_count: 3821, Cost Time: 23.08s\n",
      "Time: 2019-05-16 02:19:39.961526318~2019-05-16 02:35:30.054106322, Loss: 0.4922, Nodes_count: 3966, Cost Time: 24.40s\n",
      "Time: 2019-05-16 02:35:30.054106322~2019-05-16 02:50:35.811829269, Loss: 0.7592, Nodes_count: 4067, Cost Time: 25.97s\n",
      "Time: 2019-05-16 02:50:35.811829269~2019-05-16 03:06:05.943294327, Loss: 0.3118, Nodes_count: 4419, Cost Time: 27.75s\n",
      "Time: 2019-05-16 03:06:05.943294327~2019-05-16 03:21:47.303111250, Loss: 0.5297, Nodes_count: 4717, Cost Time: 29.14s\n",
      "Time: 2019-05-16 03:21:47.303111250~2019-05-16 03:36:53.523065535, Loss: 0.4776, Nodes_count: 4890, Cost Time: 30.28s\n",
      "Time: 2019-05-16 03:36:53.523065535~2019-05-16 03:52:06.074285645, Loss: 0.2548, Nodes_count: 5004, Cost Time: 31.01s\n",
      "Time: 2019-05-16 03:52:06.074285645~2019-05-16 04:07:28.053987688, Loss: 0.3320, Nodes_count: 5094, Cost Time: 31.94s\n",
      "Time: 2019-05-16 04:07:28.053987688~2019-05-16 04:23:52.053970386, Loss: 0.0348, Nodes_count: 5094, Cost Time: 32.43s\n",
      "Time: 2019-05-16 04:23:52.053970386~2019-05-16 04:39:16.074221006, Loss: 0.0314, Nodes_count: 5094, Cost Time: 32.85s\n",
      "Time: 2019-05-16 04:39:16.074221006~2019-05-16 04:54:40.073948996, Loss: 0.0323, Nodes_count: 5094, Cost Time: 33.26s\n",
      "Time: 2019-05-16 04:54:40.073948996~2019-05-16 05:10:12.053923923, Loss: 0.0337, Nodes_count: 5098, Cost Time: 33.72s\n",
      "Time: 2019-05-16 05:10:12.053923923~2019-05-16 05:25:30.074155632, Loss: 0.0322, Nodes_count: 5098, Cost Time: 34.14s\n",
      "Time: 2019-05-16 05:25:30.074155632~2019-05-16 05:41:08.053785658, Loss: 0.0350, Nodes_count: 5098, Cost Time: 34.55s\n",
      "Time: 2019-05-16 05:41:08.053785658~2019-05-16 05:56:36.063970033, Loss: 0.0333, Nodes_count: 5098, Cost Time: 34.96s\n",
      "Time: 2019-05-16 05:56:36.063970033~2019-05-16 06:12:56.073950599, Loss: 0.0313, Nodes_count: 5098, Cost Time: 35.43s\n",
      "Time: 2019-05-16 06:12:56.073950599~2019-05-16 06:28:28.074065777, Loss: 0.0299, Nodes_count: 5098, Cost Time: 35.84s\n",
      "Time: 2019-05-16 06:28:28.074065777~2019-05-16 06:43:56.073801568, Loss: 0.0269, Nodes_count: 5098, Cost Time: 36.25s\n",
      "Time: 2019-05-16 06:43:56.073801568~2019-05-16 06:59:18.053755875, Loss: 0.0303, Nodes_count: 5098, Cost Time: 36.67s\n",
      "Time: 2019-05-16 06:59:18.053755875~2019-05-16 07:15:28.063859188, Loss: 0.0298, Nodes_count: 5099, Cost Time: 37.13s\n",
      "Time: 2019-05-16 07:15:28.063859188~2019-05-16 07:30:46.073981043, Loss: 0.0283, Nodes_count: 5099, Cost Time: 37.55s\n",
      "Time: 2019-05-16 07:30:46.073981043~2019-05-16 07:45:56.083899517, Loss: 0.0310, Nodes_count: 5106, Cost Time: 37.96s\n",
      "Time: 2019-05-16 07:45:56.083899517~2019-05-16 08:03:04.073932182, Loss: 0.0281, Nodes_count: 5106, Cost Time: 38.43s\n",
      "Time: 2019-05-16 08:03:04.073932182~2019-05-16 08:19:48.053598991, Loss: 0.0284, Nodes_count: 5106, Cost Time: 38.90s\n",
      "Time: 2019-05-16 08:19:48.053598991~2019-05-16 08:35:02.133692641, Loss: 0.0322, Nodes_count: 5106, Cost Time: 39.31s\n",
      "Time: 2019-05-16 08:35:02.133692641~2019-05-16 08:50:04.103774736, Loss: 0.0812, Nodes_count: 5108, Cost Time: 39.73s\n",
      "Time: 2019-05-16 08:50:04.103774736~2019-05-16 09:05:18.093603996, Loss: 0.2057, Nodes_count: 5183, Cost Time: 40.86s\n",
      "Time: 2019-05-16 09:05:18.093603996~2019-05-16 09:20:32.093582942, Loss: 0.3028, Nodes_count: 5621, Cost Time: 44.79s\n",
      "Time: 2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477, Loss: 0.4509, Nodes_count: 5779, Cost Time: 48.24s\n",
      "Time: 2019-05-16 09:36:08.903494477~2019-05-16 09:51:22.110949680, Loss: 0.3715, Nodes_count: 5970, Cost Time: 50.96s\n",
      "Time: 2019-05-16 09:51:22.110949680~2019-05-16 10:06:29.403713371, Loss: 0.7709, Nodes_count: 6070, Cost Time: 53.85s\n",
      "Time: 2019-05-16 10:06:29.403713371~2019-05-16 10:21:47.983513184, Loss: 0.5386, Nodes_count: 6322, Cost Time: 57.49s\n",
      "Time: 2019-05-16 10:21:47.983513184~2019-05-16 10:37:02.053456880, Loss: 0.3824, Nodes_count: 6558, Cost Time: 62.35s\n",
      "Time: 2019-05-16 10:37:02.053456880~2019-05-16 10:52:13.133498417, Loss: 0.7056, Nodes_count: 6719, Cost Time: 67.10s\n",
      "Time: 2019-05-16 10:52:13.133498417~2019-05-16 11:07:58.483378590, Loss: 0.5143, Nodes_count: 6954, Cost Time: 70.49s\n",
      "Time: 2019-05-16 11:07:58.483378590~2019-05-16 11:23:00.723415561, Loss: 0.6208, Nodes_count: 7007, Cost Time: 75.74s\n",
      "Time: 2019-05-16 11:23:00.723415561~2019-05-16 11:38:36.270922023, Loss: 0.5199, Nodes_count: 7256, Cost Time: 78.48s\n",
      "Time: 2019-05-16 11:38:36.270922023~2019-05-16 11:54:09.383300779, Loss: 0.3101, Nodes_count: 7449, Cost Time: 81.38s\n",
      "Time: 2019-05-16 11:54:09.383300779~2019-05-16 12:09:10.073385814, Loss: 0.3898, Nodes_count: 7723, Cost Time: 86.15s\n",
      "Time: 2019-05-16 12:09:10.073385814~2019-05-16 12:24:13.482488891, Loss: 0.6760, Nodes_count: 7893, Cost Time: 89.48s\n",
      "Time: 2019-05-16 12:24:13.482488891~2019-05-16 12:39:13.512670126, Loss: 0.5513, Nodes_count: 8083, Cost Time: 95.25s\n",
      "Time: 2019-05-16 12:39:13.512670126~2019-05-16 12:54:23.383199820, Loss: 0.5324, Nodes_count: 8254, Cost Time: 100.37s\n",
      "Time: 2019-05-16 12:54:23.383199820~2019-05-16 13:10:19.070779147, Loss: 0.6679, Nodes_count: 8783, Cost Time: 106.18s\n",
      "Time: 2019-05-16 13:10:19.070779147~2019-05-16 13:25:31.470978934, Loss: 0.5842, Nodes_count: 8916, Cost Time: 110.65s\n",
      "Time: 2019-05-16 13:25:31.470978934~2019-05-16 13:40:32.073458932, Loss: 0.7251, Nodes_count: 9310, Cost Time: 117.03s\n",
      "Time: 2019-05-16 13:40:32.073458932~2019-05-16 13:55:36.560621070, Loss: 0.6112, Nodes_count: 9610, Cost Time: 124.23s\n",
      "Time: 2019-05-16 13:55:36.560621070~2019-05-16 14:10:44.950897261, Loss: 0.5484, Nodes_count: 9909, Cost Time: 130.40s\n",
      "Time: 2019-05-16 14:10:44.950897261~2019-05-16 14:25:48.270690959, Loss: 0.7154, Nodes_count: 10337, Cost Time: 136.15s\n",
      "Time: 2019-05-16 14:25:48.270690959~2019-05-16 14:41:00.063228926, Loss: 0.3825, Nodes_count: 10763, Cost Time: 143.24s\n",
      "Time: 2019-05-16 14:41:00.063228926~2019-05-16 14:56:22.063206218, Loss: 0.6794, Nodes_count: 10859, Cost Time: 145.75s\n",
      "Time: 2019-05-16 14:56:22.063206218~2019-05-16 15:11:22.163040001, Loss: 0.8840, Nodes_count: 11281, Cost Time: 150.66s\n",
      "Time: 2019-05-16 15:11:22.163040001~2019-05-16 15:27:01.353104819, Loss: 0.6205, Nodes_count: 11650, Cost Time: 155.64s\n",
      "Time: 2019-05-16 15:27:01.353104819~2019-05-16 15:42:08.532100479, Loss: 0.6215, Nodes_count: 11797, Cost Time: 158.32s\n",
      "Time: 2019-05-16 15:42:08.532100479~2019-05-16 15:57:14.083226569, Loss: 0.5865, Nodes_count: 11897, Cost Time: 160.06s\n",
      "Time: 2019-05-16 15:57:14.083226569~2019-05-16 16:12:28.333264297, Loss: 0.6314, Nodes_count: 12139, Cost Time: 164.13s\n",
      "Time: 2019-05-16 16:12:28.333264297~2019-05-16 16:27:29.952915678, Loss: 0.4836, Nodes_count: 12380, Cost Time: 167.87s\n",
      "Time: 2019-05-16 16:27:29.952915678~2019-05-16 16:42:33.013069526, Loss: 0.6000, Nodes_count: 12565, Cost Time: 170.54s\n",
      "Time: 2019-05-16 16:42:33.013069526~2019-05-16 16:57:34.092939750, Loss: 0.5089, Nodes_count: 13005, Cost Time: 177.11s\n",
      "Time: 2019-05-16 16:57:34.092939750~2019-05-16 17:13:06.280592297, Loss: 0.4258, Nodes_count: 13208, Cost Time: 180.47s\n",
      "Time: 2019-05-16 17:13:06.280592297~2019-05-16 17:28:10.223159643, Loss: 0.3566, Nodes_count: 13493, Cost Time: 186.09s\n",
      "Time: 2019-05-16 17:28:10.223159643~2019-05-16 17:43:27.572851052, Loss: 0.5395, Nodes_count: 13717, Cost Time: 190.88s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time: 2019-05-16 17:43:27.572851052~2019-05-16 17:58:44.112324274, Loss: 0.6012, Nodes_count: 13962, Cost Time: 196.92s\n",
      "Time: 2019-05-16 17:58:44.112324274~2019-05-16 18:14:14.082900415, Loss: 0.5144, Nodes_count: 14152, Cost Time: 199.97s\n",
      "Time: 2019-05-16 18:14:14.082900415~2019-05-16 18:30:26.073055443, Loss: 0.4076, Nodes_count: 14362, Cost Time: 205.31s\n",
      "Time: 2019-05-16 18:30:26.073055443~2019-05-16 18:45:49.432070287, Loss: 0.6121, Nodes_count: 14540, Cost Time: 211.26s\n",
      "Time: 2019-05-16 18:45:49.432070287~2019-05-16 19:01:12.193027191, Loss: 0.4252, Nodes_count: 14758, Cost Time: 215.03s\n",
      "Time: 2019-05-16 19:01:12.193027191~2019-05-16 19:16:22.673088266, Loss: 0.4956, Nodes_count: 14905, Cost Time: 219.03s\n",
      "Time: 2019-05-16 19:16:22.673088266~2019-05-16 19:31:32.052688344, Loss: 0.5296, Nodes_count: 15236, Cost Time: 223.12s\n",
      "Time: 2019-05-16 19:31:32.052688344~2019-05-16 19:46:40.442657625, Loss: 0.4727, Nodes_count: 15311, Cost Time: 225.76s\n",
      "Time: 2019-05-16 19:46:40.442657625~2019-05-16 20:02:06.820142338, Loss: 0.4677, Nodes_count: 15473, Cost Time: 228.46s\n",
      "Time: 2019-05-16 20:02:06.820142338~2019-05-16 20:17:11.412990296, Loss: 0.5850, Nodes_count: 15923, Cost Time: 233.28s\n",
      "Time: 2019-05-16 20:17:11.412990296~2019-05-16 20:32:27.570220441, Loss: 0.3857, Nodes_count: 16035, Cost Time: 236.57s\n",
      "Time: 2019-05-16 20:32:27.570220441~2019-05-16 20:48:38.072848659, Loss: 0.6642, Nodes_count: 16207, Cost Time: 240.67s\n",
      "Time: 2019-05-16 20:48:38.072848659~2019-05-16 21:03:58.072828936, Loss: 0.4455, Nodes_count: 16384, Cost Time: 244.19s\n",
      "Time: 2019-05-16 21:03:58.072828936~2019-05-16 21:19:00.930018779, Loss: 0.6293, Nodes_count: 16631, Cost Time: 247.30s\n",
      "Time: 2019-05-16 21:19:00.930018779~2019-05-16 21:34:46.231624861, Loss: 0.5118, Nodes_count: 16955, Cost Time: 253.41s\n",
      "Time: 2019-05-16 21:34:46.231624861~2019-05-16 21:49:46.992678639, Loss: 0.4807, Nodes_count: 17194, Cost Time: 258.41s\n",
      "Time: 2019-05-16 21:49:46.992678639~2019-05-16 22:06:14.950154813, Loss: 0.4565, Nodes_count: 17434, Cost Time: 264.27s\n",
      "Time: 2019-05-16 22:06:14.950154813~2019-05-16 22:21:40.662702391, Loss: 0.4724, Nodes_count: 17714, Cost Time: 267.32s\n",
      "Time: 2019-05-16 22:21:40.662702391~2019-05-16 22:36:45.602858389, Loss: 0.8835, Nodes_count: 18182, Cost Time: 273.89s\n",
      "Time: 2019-05-16 22:36:45.602858389~2019-05-16 22:51:51.220035024, Loss: 0.3630, Nodes_count: 18419, Cost Time: 277.68s\n",
      "Time: 2019-05-16 22:51:51.220035024~2019-05-16 23:07:16.890296254, Loss: 0.5643, Nodes_count: 18728, Cost Time: 283.54s\n",
      "Time: 2019-05-16 23:07:16.890296254~2019-05-16 23:22:54.052353000, Loss: 0.5655, Nodes_count: 19089, Cost Time: 290.00s\n",
      "Time: 2019-05-16 23:22:54.052353000~2019-05-16 23:38:22.520220953, Loss: 0.5002, Nodes_count: 20722, Cost Time: 293.62s\n",
      "Time: 2019-05-16 23:38:22.520220953~2019-05-16 23:53:23.392403039, Loss: 0.4166, Nodes_count: 21321, Cost Time: 297.86s\n"
     ]
    }
   ],
   "source": [
    "ans_5_16=test_day_new(graph_5_16,\"./graph_5_16\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after merge: TemporalData(dst=[3912303], msg=[3912303, 41], src=[3912303], t=[3912303])\n",
      "Time: 2019-05-17 00:00:00.011404169~2019-05-17 00:15:19.121412431, Loss: 0.1835, Nodes_count: 585, Cost Time: 3.87s\n",
      "Time: 2019-05-17 00:15:19.121412431~2019-05-17 00:30:22.942273426, Loss: 0.4861, Nodes_count: 1547, Cost Time: 7.30s\n",
      "Time: 2019-05-17 00:30:22.942273426~2019-05-17 00:45:36.772457892, Loss: 0.2546, Nodes_count: 1787, Cost Time: 11.62s\n",
      "Time: 2019-05-17 00:45:36.772457892~2019-05-17 01:00:43.031312698, Loss: 0.7108, Nodes_count: 2101, Cost Time: 15.03s\n",
      "Time: 2019-05-17 01:00:43.031312698~2019-05-17 01:15:44.002530354, Loss: 0.4704, Nodes_count: 2474, Cost Time: 19.18s\n",
      "Time: 2019-05-17 01:15:44.002530354~2019-05-17 01:30:47.512391580, Loss: 0.5096, Nodes_count: 2866, Cost Time: 24.26s\n",
      "Time: 2019-05-17 01:30:47.512391580~2019-05-17 01:46:06.532627237, Loss: 0.4898, Nodes_count: 2955, Cost Time: 26.52s\n",
      "Time: 2019-05-17 01:46:06.532627237~2019-05-17 02:01:06.862180143, Loss: 0.3764, Nodes_count: 3100, Cost Time: 30.39s\n",
      "Time: 2019-05-17 02:01:06.862180143~2019-05-17 02:17:12.251503432, Loss: 0.7160, Nodes_count: 3193, Cost Time: 32.92s\n",
      "Time: 2019-05-17 02:17:12.251503432~2019-05-17 02:33:34.082266094, Loss: 0.3383, Nodes_count: 3445, Cost Time: 36.76s\n",
      "Time: 2019-05-17 02:33:34.082266094~2019-05-17 02:48:48.272633335, Loss: 0.2627, Nodes_count: 3749, Cost Time: 41.75s\n",
      "Time: 2019-05-17 02:48:48.272633335~2019-05-17 03:04:02.082047814, Loss: 0.2518, Nodes_count: 3906, Cost Time: 44.59s\n",
      "Time: 2019-05-17 03:04:02.082047814~2019-05-17 03:19:06.352094009, Loss: 0.2498, Nodes_count: 4034, Cost Time: 46.89s\n",
      "Time: 2019-05-17 03:19:06.352094009~2019-05-17 03:35:22.072176911, Loss: 0.2140, Nodes_count: 4330, Cost Time: 53.56s\n",
      "Time: 2019-05-17 03:35:22.072176911~2019-05-17 03:50:24.232420260, Loss: 0.3710, Nodes_count: 4502, Cost Time: 58.19s\n",
      "Time: 2019-05-17 03:50:24.232420260~2019-05-17 04:05:38.062092796, Loss: 0.4345, Nodes_count: 4602, Cost Time: 61.17s\n",
      "Time: 2019-05-17 04:05:38.062092796~2019-05-17 04:20:39.512164044, Loss: 0.2201, Nodes_count: 4783, Cost Time: 63.58s\n",
      "Time: 2019-05-17 04:20:39.512164044~2019-05-17 04:36:14.272391092, Loss: 0.4130, Nodes_count: 4927, Cost Time: 67.51s\n",
      "Time: 2019-05-17 04:36:14.272391092~2019-05-17 04:51:18.351942306, Loss: 0.4026, Nodes_count: 5116, Cost Time: 71.17s\n",
      "Time: 2019-05-17 04:51:18.351942306~2019-05-17 05:06:35.492322242, Loss: 0.4403, Nodes_count: 5272, Cost Time: 74.07s\n",
      "Time: 2019-05-17 05:06:35.492322242~2019-05-17 05:21:53.681001911, Loss: 0.4260, Nodes_count: 5443, Cost Time: 77.69s\n",
      "Time: 2019-05-17 05:21:53.681001911~2019-05-17 05:37:20.351873432, Loss: 0.3016, Nodes_count: 5536, Cost Time: 79.53s\n",
      "Time: 2019-05-17 05:37:20.351873432~2019-05-17 05:53:46.081922263, Loss: 0.3729, Nodes_count: 5697, Cost Time: 82.54s\n",
      "Time: 2019-05-17 05:53:46.081922263~2019-05-17 06:09:26.061933299, Loss: 0.4000, Nodes_count: 5823, Cost Time: 83.64s\n",
      "Time: 2019-05-17 06:09:26.061933299~2019-05-17 06:24:48.061908897, Loss: 0.2931, Nodes_count: 5981, Cost Time: 84.64s\n",
      "Time: 2019-05-17 06:24:48.061908897~2019-05-17 06:40:14.072014912, Loss: 0.4132, Nodes_count: 6015, Cost Time: 85.37s\n",
      "Time: 2019-05-17 06:40:14.072014912~2019-05-17 06:55:30.760988446, Loss: 0.6110, Nodes_count: 6138, Cost Time: 86.58s\n",
      "Time: 2019-05-17 06:55:30.760988446~2019-05-17 07:11:16.051875332, Loss: 0.5399, Nodes_count: 6185, Cost Time: 87.65s\n",
      "Time: 2019-05-17 07:11:16.051875332~2019-05-17 07:27:34.071942527, Loss: 0.5551, Nodes_count: 6272, Cost Time: 88.43s\n",
      "Time: 2019-05-17 07:27:34.071942527~2019-05-17 07:44:04.071921360, Loss: 0.3464, Nodes_count: 6308, Cost Time: 89.41s\n",
      "Time: 2019-05-17 07:44:04.071921360~2019-05-17 07:59:22.051873504, Loss: 0.0564, Nodes_count: 6308, Cost Time: 89.85s\n",
      "Time: 2019-05-17 07:59:22.051873504~2019-05-17 08:14:56.081692880, Loss: 0.0591, Nodes_count: 6308, Cost Time: 90.31s\n",
      "Time: 2019-05-17 08:14:56.081692880~2019-05-17 08:30:22.081640414, Loss: 0.0563, Nodes_count: 6308, Cost Time: 90.73s\n",
      "Time: 2019-05-17 08:30:22.081640414~2019-05-17 08:45:38.051874367, Loss: 0.0570, Nodes_count: 6308, Cost Time: 91.14s\n",
      "Time: 2019-05-17 08:45:38.051874367~2019-05-17 09:01:32.730750122, Loss: 0.1630, Nodes_count: 6321, Cost Time: 91.61s\n",
      "Time: 2019-05-17 09:01:32.730750122~2019-05-17 09:16:44.320659825, Loss: 0.3635, Nodes_count: 6444, Cost Time: 93.70s\n",
      "Time: 2019-05-17 09:16:44.320659825~2019-05-17 09:31:47.131585900, Loss: 0.5497, Nodes_count: 6582, Cost Time: 97.74s\n",
      "Time: 2019-05-17 09:31:47.131585900~2019-05-17 09:46:50.071747334, Loss: 0.4453, Nodes_count: 6657, Cost Time: 100.93s\n",
      "Time: 2019-05-17 09:46:50.071747334~2019-05-17 10:02:11.321524261, Loss: 0.3982, Nodes_count: 6876, Cost Time: 105.60s\n",
      "Time: 2019-05-17 10:02:11.321524261~2019-05-17 10:17:26.881636687, Loss: 0.4058, Nodes_count: 7035, Cost Time: 109.95s\n",
      "Time: 2019-05-17 10:17:26.881636687~2019-05-17 10:32:38.131495470, Loss: 0.5501, Nodes_count: 7476, Cost Time: 113.99s\n",
      "Time: 2019-05-17 10:32:38.131495470~2019-05-17 10:48:02.091564015, Loss: 0.3728, Nodes_count: 7616, Cost Time: 115.70s\n",
      "Time: 2019-05-17 10:48:02.091564015~2019-05-17 11:04:49.619210641, Loss: 0.4805, Nodes_count: 8002, Cost Time: 119.78s\n",
      "Time: 2019-05-17 11:04:49.619210641~2019-05-17 11:19:49.740854239, Loss: 0.5317, Nodes_count: 8185, Cost Time: 124.81s\n",
      "Time: 2019-05-17 11:19:49.740854239~2019-05-17 11:35:22.890857355, Loss: 0.6082, Nodes_count: 8422, Cost Time: 128.29s\n",
      "Time: 2019-05-17 11:35:22.890857355~2019-05-17 11:50:25.769134767, Loss: 0.4879, Nodes_count: 8667, Cost Time: 132.13s\n",
      "Time: 2019-05-17 11:50:25.769134767~2019-05-17 12:05:34.151631198, Loss: 0.3995, Nodes_count: 9032, Cost Time: 137.69s\n",
      "Time: 2019-05-17 12:05:34.151631198~2019-05-17 12:20:48.219175447, Loss: 0.5734, Nodes_count: 9353, Cost Time: 140.36s\n",
      "Time: 2019-05-17 12:20:48.219175447~2019-05-17 12:36:20.051747107, Loss: 0.4279, Nodes_count: 9585, Cost Time: 144.54s\n",
      "Time: 2019-05-17 12:36:20.051747107~2019-05-17 12:51:28.051740428, Loss: 0.3301, Nodes_count: 9829, Cost Time: 147.09s\n",
      "Time: 2019-05-17 12:51:28.051740428~2019-05-17 13:06:40.331492736, Loss: 0.4034, Nodes_count: 9935, Cost Time: 150.32s\n",
      "Time: 2019-05-17 13:06:40.331492736~2019-05-17 13:21:54.750460761, Loss: 0.5318, Nodes_count: 10396, Cost Time: 156.35s\n",
      "Time: 2019-05-17 13:21:54.750460761~2019-05-17 13:37:08.741478317, Loss: 0.3935, Nodes_count: 10645, Cost Time: 159.48s\n",
      "Time: 2019-05-17 13:37:08.741478317~2019-05-17 13:52:20.000650535, Loss: 0.3309, Nodes_count: 10951, Cost Time: 165.15s\n",
      "Time: 2019-05-17 13:52:20.000650535~2019-05-17 14:07:34.051622439, Loss: 0.4085, Nodes_count: 11041, Cost Time: 167.74s\n",
      "Time: 2019-05-17 14:07:34.051622439~2019-05-17 14:22:39.531584558, Loss: 0.5073, Nodes_count: 11478, Cost Time: 171.13s\n",
      "Time: 2019-05-17 14:22:39.531584558~2019-05-17 14:37:39.821475906, Loss: 0.7583, Nodes_count: 11825, Cost Time: 176.88s\n",
      "Time: 2019-05-17 14:37:39.821475906~2019-05-17 14:52:40.948619952, Loss: 0.3792, Nodes_count: 12080, Cost Time: 180.53s\n",
      "Time: 2019-05-17 14:52:40.948619952~2019-05-17 15:07:56.121477590, Loss: 0.5233, Nodes_count: 12465, Cost Time: 186.50s\n",
      "Time: 2019-05-17 15:07:56.121477590~2019-05-17 15:23:16.091314858, Loss: 0.5384, Nodes_count: 12736, Cost Time: 193.59s\n",
      "Time: 2019-05-17 15:23:16.091314858~2019-05-17 15:38:50.341411090, Loss: 0.4188, Nodes_count: 12967, Cost Time: 196.06s\n",
      "Time: 2019-05-17 15:38:50.341411090~2019-05-17 15:54:28.051468310, Loss: 0.4116, Nodes_count: 13142, Cost Time: 199.50s\n",
      "Time: 2019-05-17 15:54:28.051468310~2019-05-17 16:09:30.211020620, Loss: 0.4013, Nodes_count: 13365, Cost Time: 202.82s\n",
      "Time: 2019-05-17 16:09:30.211020620~2019-05-17 16:24:30.611175860, Loss: 0.5126, Nodes_count: 13637, Cost Time: 206.68s\n",
      "Time: 2019-05-17 16:24:30.611175860~2019-05-17 16:39:33.741234335, Loss: 0.2534, Nodes_count: 13798, Cost Time: 210.87s\n",
      "Time: 2019-05-17 16:39:33.741234335~2019-05-17 16:54:49.968820600, Loss: 0.2755, Nodes_count: 13959, Cost Time: 213.09s\n",
      "Time: 2019-05-17 16:54:49.968820600~2019-05-17 17:09:59.490279010, Loss: 0.4420, Nodes_count: 14287, Cost Time: 219.42s\n",
      "Time: 2019-05-17 17:09:59.490279010~2019-05-17 17:26:04.070998589, Loss: 0.5494, Nodes_count: 14503, Cost Time: 222.41s\n",
      "Time: 2019-05-17 17:26:04.070998589~2019-05-17 17:42:18.071078452, Loss: 0.0454, Nodes_count: 14503, Cost Time: 222.82s\n"
     ]
    }
   ],
   "source": [
    "ans_5_17=test_day_new(graph_5_17,\"./graph_5_17\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialize the node IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 315/315 [03:31<00:00,  1.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IDF weight calculate complete!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "file_list=[]\n",
    "\n",
    "file_path=\"graph_5_8/\"\n",
    "file_l=os.listdir(\"graph_5_8/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)\n",
    "\n",
    "file_path=\"graph_5_9/\"\n",
    "file_l=os.listdir(\"graph_5_9/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)\n",
    "\n",
    "file_path=\"graph_5_11/\"\n",
    "file_l=os.listdir(\"graph_5_11/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)\n",
    "\n",
    "\n",
    "file_path=\"graph_5_12/\"\n",
    "file_l=os.listdir(\"graph_5_12/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)\n",
    "\n",
    "# for f_path in tqdm(file_list):\n",
    "#     f=open(f_path)\n",
    "#     for line in f:\n",
    "#         l=line.strip()\n",
    "#         jdata=eval(l)\n",
    "#         if jdata['loss']>0:\n",
    "#             if 'netflow' not in str(jdata['srcmsg']):\n",
    "#                 node_set.add(str(jdata['srcmsg']))\n",
    "#             if 'netflow' not in str(jdata['dstmsg']):\n",
    "#                 node_set.add(str(jdata['dstmsg'])) \n",
    "\n",
    "node_IDF={}\n",
    "node_set = {}\n",
    "for f_path in tqdm(file_list):\n",
    "    f=open(f_path)\n",
    "    for line in f:\n",
    "        l=line.strip()\n",
    "        jdata=eval(l)\n",
    "        jdata=eval(l)\n",
    "        if jdata['loss']>0:\n",
    "            if 'netflow' not in str(jdata['srcmsg']):\n",
    "                if str(jdata['srcmsg']) not in node_set.keys():\n",
    "                    node_set[str(jdata['srcmsg'])] = set([f_path])\n",
    "                else:\n",
    "                    node_set[str(jdata['srcmsg'])].add(f_path)\n",
    "            if 'netflow' not in str(jdata['dstmsg']):\n",
    "                if str(jdata['dstmsg']) not in node_set.keys():\n",
    "                    node_set[str(jdata['dstmsg'])] = set([f_path])\n",
    "                else:\n",
    "                    node_set[str(jdata['dstmsg'])].add(f_path)\n",
    "for n in node_set:\n",
    "    include_count = len(node_set[n])   \n",
    "    IDF=math.log(len(file_list)/(include_count+1))\n",
    "    node_IDF[n] = IDF    \n",
    "\n",
    "\n",
    "torch.save(node_IDF,\"node_IDF\")\n",
    "print(\"IDF weight calculate complete!\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cal_train_IDF(find_str,file_list):\n",
    "    include_count=0\n",
    "    for f_path in (file_list):\n",
    "        f=open(f_path)\n",
    "        if find_str in f.read():\n",
    "            include_count+=1             \n",
    "    IDF=math.log(len(file_list)/(include_count+1))\n",
    "    return IDF\n",
    "\n",
    "\n",
    "def cal_IDF(find_str,file_path,file_list):\n",
    "    file_list=os.listdir(file_path)\n",
    "    include_count=0\n",
    "    different_neighbor=set()\n",
    "    for f_path in (file_list):\n",
    "        f=open(file_path+f_path)\n",
    "        if find_str in f.read():\n",
    "            include_count+=1                \n",
    "                \n",
    "    IDF=math.log(len(file_list)/(include_count+1))\n",
    "    \n",
    "    return IDF,1\n",
    "\n",
    "def cal_redundant(find_str,edge_list):\n",
    "    \n",
    "    different_neighbor=set()\n",
    "    for e in edge_list:\n",
    "        if find_str in str(e):\n",
    "            different_neighbor.add(e[0])\n",
    "            different_neighbor.add(e[1])\n",
    "    return len(different_neighbor)-2\n",
    "\n",
    "def cal_anomaly_loss(loss_list,edge_list,file_path):\n",
    "    \n",
    "    if len(loss_list)!=len(edge_list):\n",
    "        print(\"error!\")\n",
    "        return 0\n",
    "    count=0\n",
    "    loss_sum=0\n",
    "    loss_std=std(loss_list)\n",
    "    loss_mean=mean(loss_list)\n",
    "    edge_set=set()\n",
    "    node_set=set()\n",
    "    node2redundant={}\n",
    "    \n",
    "    thr=loss_mean+1.5*loss_std\n",
    "\n",
    "    print(\"thr:\",thr)\n",
    "\n",
    "    for i in range(len(loss_list)):\n",
    "        if loss_list[i]>thr:\n",
    "            count+=1\n",
    "            src_node=edge_list[i][0]\n",
    "            dst_node=edge_list[i][1]\n",
    "            \n",
    "            loss_sum+=loss_list[i]\n",
    "    \n",
    "            node_set.add(src_node)\n",
    "            node_set.add(dst_node)\n",
    "            edge_set.add(edge_list[i][0]+edge_list[i][1])\n",
    "    return count, loss_sum/count,node_set,edge_set\n",
    "#     return count, count/len(loss_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Construct the relations between time windows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # node_IDF_5_9_12=torch.load(\"node_IDF_5_9-12\")\n",
    "# def cal_set_rel(s1,s2,file_list):\n",
    "#     new_s=s1 & s2\n",
    "#     count=0\n",
    "#     for i in new_s:\n",
    "# #     jdata=json.loads(i)\n",
    "#         if 'netflow' not in i and '/home/admin/' not in i and  '/proc/' not in i :\n",
    "        \n",
    "# #         'netflow' not in i\n",
    "# #         and 'usr' not in i and 'var' not in i\n",
    "#             if i in node_IDF.keys():\n",
    "#                 IDF=node_IDF[i]\n",
    "#             else:\n",
    "#                 IDF=math.log(len(file_list)/(1))\n",
    "                \n",
    "#             if i in node_IDF_4_4_7.keys():\n",
    "#                 IDF4=node_IDF_4_4_7[i]\n",
    "#             else:\n",
    "#                 IDF4=math.log(len(file_list_4_4_7)/(1))    \n",
    "            \n",
    "# #             print(IDF)\n",
    "#             if (IDF+IDF4)>9:\n",
    "#                 print(\"node:\",i,\" IDF:\",IDF)\n",
    "#                 count+=1\n",
    "#     return count\n",
    "\n",
    "\n",
    "# def cal_set_rel_bak(s1,s2,file_list):\n",
    "#     new_s=s1 & s2\n",
    "#     count=0\n",
    "#     for i in new_s:\n",
    "# #     jdata=json.loads(i)\n",
    "#         if 'netflow' not in i \\\n",
    "#     and '/home/admin/' not in i \\\n",
    "#             and '/home/user/' not in i\\\n",
    "#             and '/tmp/' not in i\\\n",
    "#     and '/tmp/refload_pStageMem_log' not in i\\\n",
    "#             and 'com.' not in i:\n",
    "        \n",
    "# #         and '.dziauz.tinyflashlight' not in i \\\n",
    "# #             and '/data/system_ce/ not in i \\\n",
    "            \n",
    "# #         and 'usr' not in i and 'proc' not in i and '675' not in i and 'firefox' not in i and 'tmp' not in i and 'thunderbird' not in i\n",
    "# #         'netflow' not in i\n",
    "# #         and 'usr' not in i and 'var' not in i\n",
    "#             if i in node_IDF.keys():\n",
    "#                 IDF=node_IDF[i]\n",
    "#             else:\n",
    "#                 IDF=math.log(len(file_list)/(1))           \n",
    "                   \n",
    "# #             print(IDF)\n",
    "# #             print(len(file_list))\n",
    "#             if IDF>math.log(len(file_list)*0.9/(1))  :\n",
    "#                 print(\"node:\",i,\" IDF:\",IDF)\n",
    "#                 count+=1\n",
    "#     return count\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def is_include_key_word_bak(s):\n",
    "    keywords=[\n",
    "         'netflow',\n",
    "\n",
    "        '/home/admin/',\n",
    "        '/home/user/',\n",
    "         'proc',\n",
    "        '/tmp/',\n",
    "        '/var/spool/mqueue/',\n",
    "        '/var/log/debug.log.0',\n",
    "      \n",
    "      ]\n",
    "    flag=False\n",
    "    for i in keywords:\n",
    "        if i in s:\n",
    "            flag=True\n",
    "    return flag\n",
    "\n",
    "\n",
    "def cal_set_rel_bak(s1,s2,file_list):\n",
    "    new_s=s1 & s2\n",
    "    count=0\n",
    "    for i in new_s:\n",
    "        if is_include_key_word_bak(i) is not True:\n",
    "            if i in node_IDF.keys():\n",
    "                IDF=node_IDF[i]\n",
    "            else:\n",
    "                IDF=math.log(len(file_list)/(1))           \n",
    "                   \n",
    "            if (IDF)>math.log(len(file_list)*0.9/(1))  :\n",
    "                print(\"node:\",i,\" IDF:\",IDF)\n",
    "                count+=1\n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_label={}   \n",
    "\n",
    "filelist = os.listdir(\"./graph_5_15\")\n",
    "for f in filelist:\n",
    "\n",
    "    pred_label[\"./graph_5_15/\"+f]=0\n",
    "    \n",
    "filelist = os.listdir(\"./graph_5_16\")\n",
    "for f in filelist:\n",
    "\n",
    "    pred_label[\"./graph_5_16/\"+f]=0\n",
    "    \n",
    "filelist = os.listdir(\"./graph_5_17\")\n",
    "for f in filelist:\n",
    "    pred_label[\"./graph_5_17/\"+f]=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'./graph_5_15/2019-05-15 10:03:41.465554650~2019-05-15 10:18:48.315349922.txt': 0,\n",
       " './graph_5_15/2019-05-15 15:11:29.355153391~2019-05-15 15:26:30.315069987.txt': 0,\n",
       " './graph_5_15/2019-05-15 14:40:58.314993755~2019-05-15 14:56:22.325270164.txt': 0,\n",
       " './graph_5_15/2019-05-15 14:56:22.325270164~2019-05-15 15:11:29.355153391.txt': 0,\n",
       " './graph_5_15/2019-05-15 23:04:18.584315206~2019-05-15 23:19:21.464426330.txt': 0,\n",
       " './graph_5_15/2019-05-15 21:03:34.932076311~2019-05-15 21:18:36.594847325.txt': 0,\n",
       " './graph_5_15/2019-05-15 00:15:13.743767966~2019-05-15 00:30:15.346246304.txt': 0,\n",
       " './graph_5_15/2019-05-15 00:45:38.935375235~2019-05-15 01:00:49.986348448.txt': 0,\n",
       " './graph_5_15/2019-05-15 18:30:12.692563554~2019-05-15 18:45:36.312255908.txt': 0,\n",
       " './graph_5_15/2019-05-15 07:39:16.336018368~2019-05-15 07:55:36.325862117.txt': 0,\n",
       " './graph_5_15/2019-05-15 10:34:18.902982618~2019-05-15 10:49:19.662777306.txt': 0,\n",
       " './graph_5_15/2019-05-15 08:46:56.325786951~2019-05-15 09:02:05.234725465.txt': 0,\n",
       " './graph_5_15/2019-05-15 12:53:47.114274288~2019-05-15 13:08:48.445537909.txt': 0,\n",
       " './graph_5_15/2019-05-15 15:26:30.315069987~2019-05-15 15:41:58.325198150.txt': 0,\n",
       " './graph_5_15/2019-05-15 17:44:50.054854387~2019-05-15 17:59:50.764934674.txt': 0,\n",
       " './graph_5_15/2019-05-15 22:49:14.843539280~2019-05-15 23:04:18.584315206.txt': 0,\n",
       " './graph_5_15/2019-05-15 01:46:35.756168353~2019-05-15 02:02:03.445171413.txt': 0,\n",
       " './graph_5_15/2019-05-15 10:18:48.315349922~2019-05-15 10:34:18.902982618.txt': 0,\n",
       " './graph_5_15/2019-05-15 16:12:04.095060867~2019-05-15 16:27:36.075256581.txt': 0,\n",
       " './graph_5_15/2019-05-15 06:00:28.326014326~2019-05-15 06:17:32.325991084.txt': 0,\n",
       " './graph_5_15/2019-05-15 06:33:44.315665234~2019-05-15 06:49:36.336078067.txt': 0,\n",
       " './graph_5_15/2019-05-15 22:34:04.714368495~2019-05-15 22:49:14.843539280.txt': 0,\n",
       " './graph_5_15/2019-05-15 07:55:36.325862117~2019-05-15 08:12:44.335538340.txt': 0,\n",
       " './graph_5_15/2019-05-15 20:33:20.173943586~2019-05-15 20:48:26.344519703.txt': 0,\n",
       " './graph_5_15/2019-05-15 11:04:34.935316812~2019-05-15 11:20:14.345345440.txt': 0,\n",
       " './graph_5_15/2019-05-15 13:39:04.835250858~2019-05-15 13:54:20.623015847.txt': 0,\n",
       " './graph_5_15/2019-05-15 18:45:36.312255908~2019-05-15 19:00:43.932284310.txt': 0,\n",
       " './graph_5_15/2019-05-15 07:05:22.335601187~2019-05-15 07:23:00.336037899.txt': 0,\n",
       " './graph_5_15/2019-05-15 11:51:01.435516598~2019-05-15 12:07:06.345256228.txt': 0,\n",
       " './graph_5_15/2019-05-15 05:44:40.326005052~2019-05-15 06:00:28.326014326.txt': 0,\n",
       " './graph_5_15/2019-05-15 03:04:19.506191624~2019-05-15 03:19:20.336382785.txt': 0,\n",
       " './graph_5_15/2019-05-15 21:33:37.744873139~2019-05-15 21:48:38.194423026.txt': 0,\n",
       " './graph_5_15/2019-05-15 01:31:15.876102508~2019-05-15 01:46:35.756168353.txt': 0,\n",
       " './graph_5_15/2019-05-15 14:25:00.732554067~2019-05-15 14:40:58.314993755.txt': 0,\n",
       " './graph_5_15/2019-05-15 11:35:14.525271329~2019-05-15 11:51:01.435516598.txt': 0,\n",
       " './graph_5_15/2019-05-15 02:33:08.316023927~2019-05-15 02:48:12.346047654.txt': 0,\n",
       " './graph_5_15/2019-05-15 03:34:24.565949954~2019-05-15 03:50:08.335957105.txt': 0,\n",
       " './graph_5_15/2019-05-15 20:02:51.452177789~2019-05-15 20:18:20.054640718.txt': 0,\n",
       " './graph_5_15/2019-05-15 12:38:28.335591405~2019-05-15 12:53:47.114274288.txt': 0,\n",
       " './graph_5_15/2019-05-15 13:54:20.623015847~2019-05-15 14:09:38.985029953.txt': 0,\n",
       " './graph_5_15/2019-05-15 16:43:24.282499513~2019-05-15 16:58:34.225243276.txt': 0,\n",
       " './graph_5_15/2019-05-15 03:19:20.336382785~2019-05-15 03:34:24.565949954.txt': 0,\n",
       " './graph_5_15/2019-05-15 02:17:36.366097118~2019-05-15 02:33:08.316023927.txt': 0,\n",
       " './graph_5_15/2019-05-15 08:29:08.335580692~2019-05-15 08:46:56.325786951.txt': 0,\n",
       " './graph_5_15/2019-05-15 02:02:03.445171413~2019-05-15 02:17:36.366097118.txt': 0,\n",
       " './graph_5_15/2019-05-15 01:00:49.986348448~2019-05-15 01:16:13.516335500.txt': 0,\n",
       " './graph_5_15/2019-05-15 19:47:26.074698234~2019-05-15 20:02:51.452177789.txt': 0,\n",
       " './graph_5_15/2019-05-15 12:22:23.514541754~2019-05-15 12:38:28.335591405.txt': 0,\n",
       " './graph_5_15/2019-05-15 20:48:26.344519703~2019-05-15 21:03:34.932076311.txt': 0,\n",
       " './graph_5_15/2019-05-15 17:28:46.064963493~2019-05-15 17:44:50.054854387.txt': 0,\n",
       " './graph_5_15/2019-05-15 19:32:22.054614132~2019-05-15 19:47:26.074698234.txt': 0,\n",
       " './graph_5_15/2019-05-15 13:08:48.445537909~2019-05-15 13:23:52.255450878.txt': 0,\n",
       " './graph_5_15/2019-05-15 07:23:00.336037899~2019-05-15 07:39:16.336018368.txt': 0,\n",
       " './graph_5_15/2019-05-15 00:00:00.006192320~2019-05-15 00:15:13.743767966.txt': 0,\n",
       " './graph_5_15/2019-05-15 14:09:38.985029953~2019-05-15 14:25:00.732554067.txt': 0,\n",
       " './graph_5_15/2019-05-15 22:19:00.063809691~2019-05-15 22:34:04.714368495.txt': 0,\n",
       " './graph_5_15/2019-05-15 23:34:24.683386847~2019-05-15 23:49:33.091873325.txt': 0,\n",
       " './graph_5_15/2019-05-15 17:59:50.764934674~2019-05-15 18:15:04.084749724.txt': 0,\n",
       " './graph_5_15/2019-05-15 17:13:45.412245202~2019-05-15 17:28:46.064963493.txt': 0,\n",
       " './graph_5_15/2019-05-15 16:58:34.225243276~2019-05-15 17:13:45.412245202.txt': 0,\n",
       " './graph_5_15/2019-05-15 01:16:13.516335500~2019-05-15 01:31:15.876102508.txt': 0,\n",
       " './graph_5_15/2019-05-15 09:32:11.065706817~2019-05-15 09:48:38.684621579.txt': 0,\n",
       " './graph_5_15/2019-05-15 02:48:12.346047654~2019-05-15 03:04:19.506191624.txt': 0,\n",
       " './graph_5_15/2019-05-15 16:27:36.075256581~2019-05-15 16:43:24.282499513.txt': 0,\n",
       " './graph_5_15/2019-05-15 09:17:07.225888229~2019-05-15 09:32:11.065706817.txt': 0,\n",
       " './graph_5_15/2019-05-15 18:15:04.084749724~2019-05-15 18:30:12.692563554.txt': 0,\n",
       " './graph_5_15/2019-05-15 05:28:36.315778004~2019-05-15 05:44:40.326005052.txt': 0,\n",
       " './graph_5_15/2019-05-15 15:56:59.265333194~2019-05-15 16:12:04.095060867.txt': 0,\n",
       " './graph_5_15/2019-05-15 13:23:52.255450878~2019-05-15 13:39:04.835250858.txt': 0,\n",
       " './graph_5_15/2019-05-15 05:13:08.315787838~2019-05-15 05:28:36.315778004.txt': 0,\n",
       " './graph_5_15/2019-05-15 22:03:43.404738808~2019-05-15 22:19:00.063809691.txt': 0,\n",
       " './graph_5_15/2019-05-15 04:56:36.326111458~2019-05-15 05:13:08.315787838.txt': 0,\n",
       " './graph_5_15/2019-05-15 19:16:40.834769346~2019-05-15 19:32:22.054614132.txt': 0,\n",
       " './graph_5_15/2019-05-15 04:25:32.335891238~2019-05-15 04:41:16.315832364.txt': 0,\n",
       " './graph_5_15/2019-05-15 06:49:36.336078067~2019-05-15 07:05:22.335601187.txt': 0,\n",
       " './graph_5_15/2019-05-15 10:49:19.662777306~2019-05-15 11:04:34.935316812.txt': 0,\n",
       " './graph_5_15/2019-05-15 11:20:14.345345440~2019-05-15 11:35:14.525271329.txt': 0,\n",
       " './graph_5_15/2019-05-15 20:18:20.054640718~2019-05-15 20:33:20.173943586.txt': 0,\n",
       " './graph_5_15/2019-05-15 12:07:06.345256228~2019-05-15 12:22:23.514541754.txt': 0,\n",
       " './graph_5_15/2019-05-15 21:48:38.194423026~2019-05-15 22:03:43.404738808.txt': 0,\n",
       " './graph_5_15/2019-05-15 03:50:08.335957105~2019-05-15 04:07:57.715926628.txt': 0,\n",
       " './graph_5_15/2019-05-15 08:12:44.335538340~2019-05-15 08:29:08.335580692.txt': 0,\n",
       " './graph_5_15/2019-05-15 23:19:21.464426330~2019-05-15 23:34:24.683386847.txt': 0,\n",
       " './graph_5_15/2019-05-15 04:07:57.715926628~2019-05-15 04:25:32.335891238.txt': 0,\n",
       " './graph_5_15/2019-05-15 21:18:36.594847325~2019-05-15 21:33:37.744873139.txt': 0,\n",
       " './graph_5_15/2019-05-15 09:02:05.234725465~2019-05-15 09:17:07.225888229.txt': 0,\n",
       " './graph_5_15/2019-05-15 15:41:58.325198150~2019-05-15 15:56:59.265333194.txt': 0,\n",
       " './graph_5_15/2019-05-15 00:30:15.346246304~2019-05-15 00:45:38.935375235.txt': 0,\n",
       " './graph_5_15/2019-05-15 19:00:43.932284310~2019-05-15 19:16:40.834769346.txt': 0,\n",
       " './graph_5_15/2019-05-15 09:48:38.684621579~2019-05-15 10:03:41.465554650.txt': 0,\n",
       " './graph_5_15/2019-05-15 06:17:32.325991084~2019-05-15 06:33:44.315665234.txt': 0,\n",
       " './graph_5_15/2019-05-15 04:41:16.315832364~2019-05-15 04:56:36.326111458.txt': 0,\n",
       " './graph_5_16/2019-05-16 17:28:10.223159643~2019-05-16 17:43:27.572851052.txt': 0,\n",
       " './graph_5_16/2019-05-16 19:31:32.052688344~2019-05-16 19:46:40.442657625.txt': 0,\n",
       " './graph_5_16/2019-05-16 12:09:10.073385814~2019-05-16 12:24:13.482488891.txt': 0,\n",
       " './graph_5_16/2019-05-16 08:03:04.073932182~2019-05-16 08:19:48.053598991.txt': 0,\n",
       " './graph_5_16/2019-05-16 11:38:36.270922023~2019-05-16 11:54:09.383300779.txt': 0,\n",
       " './graph_5_16/2019-05-16 04:07:28.053987688~2019-05-16 04:23:52.053970386.txt': 0,\n",
       " './graph_5_16/2019-05-16 14:56:22.063206218~2019-05-16 15:11:22.163040001.txt': 0,\n",
       " './graph_5_16/2019-05-16 16:57:34.092939750~2019-05-16 17:13:06.280592297.txt': 0,\n",
       " './graph_5_16/2019-05-16 01:17:54.074506572~2019-05-16 01:32:54.901798763.txt': 0,\n",
       " './graph_5_16/2019-05-16 21:49:46.992678639~2019-05-16 22:06:14.950154813.txt': 0,\n",
       " './graph_5_16/2019-05-16 04:23:52.053970386~2019-05-16 04:39:16.074221006.txt': 0,\n",
       " './graph_5_16/2019-05-16 09:05:18.093603996~2019-05-16 09:20:32.093582942.txt': 0,\n",
       " './graph_5_16/2019-05-16 23:38:22.520220953~2019-05-16 23:53:23.392403039.txt': 0,\n",
       " './graph_5_16/2019-05-16 15:57:14.083226569~2019-05-16 16:12:28.333264297.txt': 0,\n",
       " './graph_5_16/2019-05-16 21:19:00.930018779~2019-05-16 21:34:46.231624861.txt': 0,\n",
       " './graph_5_16/2019-05-16 08:50:04.103774736~2019-05-16 09:05:18.093603996.txt': 0,\n",
       " './graph_5_16/2019-05-16 16:12:28.333264297~2019-05-16 16:27:29.952915678.txt': 0,\n",
       " './graph_5_16/2019-05-16 23:07:16.890296254~2019-05-16 23:22:54.052353000.txt': 0,\n",
       " './graph_5_16/2019-05-16 15:11:22.163040001~2019-05-16 15:27:01.353104819.txt': 0,\n",
       " './graph_5_16/2019-05-16 12:24:13.482488891~2019-05-16 12:39:13.512670126.txt': 0,\n",
       " './graph_5_16/2019-05-16 03:21:47.303111250~2019-05-16 03:36:53.523065535.txt': 0,\n",
       " './graph_5_16/2019-05-16 02:03:13.643316639~2019-05-16 02:19:39.961526318.txt': 0,\n",
       " './graph_5_16/2019-05-16 01:48:08.801926929~2019-05-16 02:03:13.643316639.txt': 0,\n",
       " './graph_5_16/2019-05-16 05:56:36.063970033~2019-05-16 06:12:56.073950599.txt': 0,\n",
       " './graph_5_16/2019-05-16 05:41:08.053785658~2019-05-16 05:56:36.063970033.txt': 0,\n",
       " './graph_5_16/2019-05-16 20:48:38.072848659~2019-05-16 21:03:58.072828936.txt': 0,\n",
       " './graph_5_16/2019-05-16 19:01:12.193027191~2019-05-16 19:16:22.673088266.txt': 0,\n",
       " './graph_5_16/2019-05-16 07:15:28.063859188~2019-05-16 07:30:46.073981043.txt': 0,\n",
       " './graph_5_16/2019-05-16 00:15:02.554575844~2019-05-16 00:30:19.324454842.txt': 0,\n",
       " './graph_5_16/2019-05-16 04:39:16.074221006~2019-05-16 04:54:40.073948996.txt': 0,\n",
       " './graph_5_16/2019-05-16 21:03:58.072828936~2019-05-16 21:19:00.930018779.txt': 0,\n",
       " './graph_5_16/2019-05-16 09:36:08.903494477~2019-05-16 09:51:22.110949680.txt': 0,\n",
       " './graph_5_16/2019-05-16 22:06:14.950154813~2019-05-16 22:21:40.662702391.txt': 0,\n",
       " './graph_5_16/2019-05-16 18:45:49.432070287~2019-05-16 19:01:12.193027191.txt': 0,\n",
       " './graph_5_16/2019-05-16 13:55:36.560621070~2019-05-16 14:10:44.950897261.txt': 0,\n",
       " './graph_5_16/2019-05-16 08:19:48.053598991~2019-05-16 08:35:02.133692641.txt': 0,\n",
       " './graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt': 0,\n",
       " './graph_5_16/2019-05-16 00:00:00.014304910~2019-05-16 00:15:02.554575844.txt': 0,\n",
       " './graph_5_16/2019-05-16 12:39:13.512670126~2019-05-16 12:54:23.383199820.txt': 0,\n",
       " './graph_5_16/2019-05-16 10:21:47.983513184~2019-05-16 10:37:02.053456880.txt': 0,\n",
       " './graph_5_16/2019-05-16 10:37:02.053456880~2019-05-16 10:52:13.133498417.txt': 0,\n",
       " './graph_5_16/2019-05-16 17:43:27.572851052~2019-05-16 17:58:44.112324274.txt': 0,\n",
       " './graph_5_16/2019-05-16 19:46:40.442657625~2019-05-16 20:02:06.820142338.txt': 0,\n",
       " './graph_5_16/2019-05-16 00:45:31.114609140~2019-05-16 01:01:06.524611321.txt': 0,\n",
       " './graph_5_16/2019-05-16 17:13:06.280592297~2019-05-16 17:28:10.223159643.txt': 0,\n",
       " './graph_5_16/2019-05-16 06:28:28.074065777~2019-05-16 06:43:56.073801568.txt': 0,\n",
       " './graph_5_16/2019-05-16 18:30:26.073055443~2019-05-16 18:45:49.432070287.txt': 0,\n",
       " './graph_5_16/2019-05-16 01:01:06.524611321~2019-05-16 01:17:54.074506572.txt': 0,\n",
       " './graph_5_16/2019-05-16 07:30:46.073981043~2019-05-16 07:45:56.083899517.txt': 0,\n",
       " './graph_5_16/2019-05-16 08:35:02.133692641~2019-05-16 08:50:04.103774736.txt': 0,\n",
       " './graph_5_16/2019-05-16 15:42:08.532100479~2019-05-16 15:57:14.083226569.txt': 0,\n",
       " './graph_5_16/2019-05-16 19:16:22.673088266~2019-05-16 19:31:32.052688344.txt': 0,\n",
       " './graph_5_16/2019-05-16 06:59:18.053755875~2019-05-16 07:15:28.063859188.txt': 0,\n",
       " './graph_5_16/2019-05-16 16:27:29.952915678~2019-05-16 16:42:33.013069526.txt': 0,\n",
       " './graph_5_16/2019-05-16 07:45:56.083899517~2019-05-16 08:03:04.073932182.txt': 0,\n",
       " './graph_5_16/2019-05-16 01:32:54.901798763~2019-05-16 01:48:08.801926929.txt': 0,\n",
       " './graph_5_16/2019-05-16 02:35:30.054106322~2019-05-16 02:50:35.811829269.txt': 0,\n",
       " './graph_5_16/2019-05-16 16:42:33.013069526~2019-05-16 16:57:34.092939750.txt': 0,\n",
       " './graph_5_16/2019-05-16 20:17:11.412990296~2019-05-16 20:32:27.570220441.txt': 0,\n",
       " './graph_5_16/2019-05-16 14:10:44.950897261~2019-05-16 14:25:48.270690959.txt': 0,\n",
       " './graph_5_16/2019-05-16 22:36:45.602858389~2019-05-16 22:51:51.220035024.txt': 0,\n",
       " './graph_5_16/2019-05-16 02:50:35.811829269~2019-05-16 03:06:05.943294327.txt': 0,\n",
       " './graph_5_16/2019-05-16 09:51:22.110949680~2019-05-16 10:06:29.403713371.txt': 0,\n",
       " './graph_5_16/2019-05-16 02:19:39.961526318~2019-05-16 02:35:30.054106322.txt': 0,\n",
       " './graph_5_16/2019-05-16 06:12:56.073950599~2019-05-16 06:28:28.074065777.txt': 0,\n",
       " './graph_5_16/2019-05-16 15:27:01.353104819~2019-05-16 15:42:08.532100479.txt': 0,\n",
       " './graph_5_16/2019-05-16 11:54:09.383300779~2019-05-16 12:09:10.073385814.txt': 0,\n",
       " './graph_5_16/2019-05-16 12:54:23.383199820~2019-05-16 13:10:19.070779147.txt': 0,\n",
       " './graph_5_16/2019-05-16 13:25:31.470978934~2019-05-16 13:40:32.073458932.txt': 0,\n",
       " './graph_5_16/2019-05-16 00:30:19.324454842~2019-05-16 00:45:31.114609140.txt': 0,\n",
       " './graph_5_16/2019-05-16 22:51:51.220035024~2019-05-16 23:07:16.890296254.txt': 0,\n",
       " './graph_5_16/2019-05-16 20:32:27.570220441~2019-05-16 20:48:38.072848659.txt': 0,\n",
       " './graph_5_16/2019-05-16 10:06:29.403713371~2019-05-16 10:21:47.983513184.txt': 0,\n",
       " './graph_5_16/2019-05-16 23:22:54.052353000~2019-05-16 23:38:22.520220953.txt': 0,\n",
       " './graph_5_16/2019-05-16 10:52:13.133498417~2019-05-16 11:07:58.483378590.txt': 0,\n",
       " './graph_5_16/2019-05-16 05:25:30.074155632~2019-05-16 05:41:08.053785658.txt': 0,\n",
       " './graph_5_16/2019-05-16 17:58:44.112324274~2019-05-16 18:14:14.082900415.txt': 0,\n",
       " './graph_5_16/2019-05-16 11:07:58.483378590~2019-05-16 11:23:00.723415561.txt': 0,\n",
       " './graph_5_16/2019-05-16 05:10:12.053923923~2019-05-16 05:25:30.074155632.txt': 0,\n",
       " './graph_5_16/2019-05-16 18:14:14.082900415~2019-05-16 18:30:26.073055443.txt': 0,\n",
       " './graph_5_16/2019-05-16 03:36:53.523065535~2019-05-16 03:52:06.074285645.txt': 0,\n",
       " './graph_5_16/2019-05-16 13:40:32.073458932~2019-05-16 13:55:36.560621070.txt': 0,\n",
       " './graph_5_16/2019-05-16 11:23:00.723415561~2019-05-16 11:38:36.270922023.txt': 0,\n",
       " './graph_5_16/2019-05-16 03:06:05.943294327~2019-05-16 03:21:47.303111250.txt': 0,\n",
       " './graph_5_16/2019-05-16 14:25:48.270690959~2019-05-16 14:41:00.063228926.txt': 0,\n",
       " './graph_5_16/2019-05-16 04:54:40.073948996~2019-05-16 05:10:12.053923923.txt': 0,\n",
       " './graph_5_16/2019-05-16 14:41:00.063228926~2019-05-16 14:56:22.063206218.txt': 0,\n",
       " './graph_5_16/2019-05-16 03:52:06.074285645~2019-05-16 04:07:28.053987688.txt': 0,\n",
       " './graph_5_16/2019-05-16 06:43:56.073801568~2019-05-16 06:59:18.053755875.txt': 0,\n",
       " './graph_5_16/2019-05-16 20:02:06.820142338~2019-05-16 20:17:11.412990296.txt': 0,\n",
       " './graph_5_16/2019-05-16 13:10:19.070779147~2019-05-16 13:25:31.470978934.txt': 0,\n",
       " './graph_5_16/2019-05-16 22:21:40.662702391~2019-05-16 22:36:45.602858389.txt': 0,\n",
       " './graph_5_16/2019-05-16 21:34:46.231624861~2019-05-16 21:49:46.992678639.txt': 0,\n",
       " './graph_5_17/2019-05-17 04:20:39.512164044~2019-05-17 04:36:14.272391092.txt': 0,\n",
       " './graph_5_17/2019-05-17 06:55:30.760988446~2019-05-17 07:11:16.051875332.txt': 0,\n",
       " './graph_5_17/2019-05-17 01:46:06.532627237~2019-05-17 02:01:06.862180143.txt': 0,\n",
       " './graph_5_17/2019-05-17 16:09:30.211020620~2019-05-17 16:24:30.611175860.txt': 0,\n",
       " './graph_5_17/2019-05-17 01:00:43.031312698~2019-05-17 01:15:44.002530354.txt': 0,\n",
       " './graph_5_17/2019-05-17 11:19:49.740854239~2019-05-17 11:35:22.890857355.txt': 0,\n",
       " './graph_5_17/2019-05-17 10:48:02.091564015~2019-05-17 11:04:49.619210641.txt': 0,\n",
       " './graph_5_17/2019-05-17 01:30:47.512391580~2019-05-17 01:46:06.532627237.txt': 0,\n",
       " './graph_5_17/2019-05-17 05:37:20.351873432~2019-05-17 05:53:46.081922263.txt': 0,\n",
       " './graph_5_17/2019-05-17 02:01:06.862180143~2019-05-17 02:17:12.251503432.txt': 0,\n",
       " './graph_5_17/2019-05-17 10:32:38.131495470~2019-05-17 10:48:02.091564015.txt': 0,\n",
       " './graph_5_17/2019-05-17 01:15:44.002530354~2019-05-17 01:30:47.512391580.txt': 0,\n",
       " './graph_5_17/2019-05-17 07:44:04.071921360~2019-05-17 07:59:22.051873504.txt': 0,\n",
       " './graph_5_17/2019-05-17 14:22:39.531584558~2019-05-17 14:37:39.821475906.txt': 0,\n",
       " './graph_5_17/2019-05-17 03:19:06.352094009~2019-05-17 03:35:22.072176911.txt': 0,\n",
       " './graph_5_17/2019-05-17 14:37:39.821475906~2019-05-17 14:52:40.948619952.txt': 0,\n",
       " './graph_5_17/2019-05-17 05:06:35.492322242~2019-05-17 05:21:53.681001911.txt': 0,\n",
       " './graph_5_17/2019-05-17 12:36:20.051747107~2019-05-17 12:51:28.051740428.txt': 0,\n",
       " './graph_5_17/2019-05-17 00:15:19.121412431~2019-05-17 00:30:22.942273426.txt': 0,\n",
       " './graph_5_17/2019-05-17 09:16:44.320659825~2019-05-17 09:31:47.131585900.txt': 0,\n",
       " './graph_5_17/2019-05-17 06:24:48.061908897~2019-05-17 06:40:14.072014912.txt': 0,\n",
       " './graph_5_17/2019-05-17 16:54:49.968820600~2019-05-17 17:09:59.490279010.txt': 0,\n",
       " './graph_5_17/2019-05-17 04:51:18.351942306~2019-05-17 05:06:35.492322242.txt': 0,\n",
       " './graph_5_17/2019-05-17 15:38:50.341411090~2019-05-17 15:54:28.051468310.txt': 0,\n",
       " './graph_5_17/2019-05-17 02:48:48.272633335~2019-05-17 03:04:02.082047814.txt': 0,\n",
       " './graph_5_17/2019-05-17 15:23:16.091314858~2019-05-17 15:38:50.341411090.txt': 0,\n",
       " './graph_5_17/2019-05-17 15:54:28.051468310~2019-05-17 16:09:30.211020620.txt': 0,\n",
       " './graph_5_17/2019-05-17 13:21:54.750460761~2019-05-17 13:37:08.741478317.txt': 0,\n",
       " './graph_5_17/2019-05-17 06:09:26.061933299~2019-05-17 06:24:48.061908897.txt': 0,\n",
       " './graph_5_17/2019-05-17 05:21:53.681001911~2019-05-17 05:37:20.351873432.txt': 0,\n",
       " './graph_5_17/2019-05-17 10:17:26.881636687~2019-05-17 10:32:38.131495470.txt': 0,\n",
       " './graph_5_17/2019-05-17 14:07:34.051622439~2019-05-17 14:22:39.531584558.txt': 0,\n",
       " './graph_5_17/2019-05-17 04:05:38.062092796~2019-05-17 04:20:39.512164044.txt': 0,\n",
       " './graph_5_17/2019-05-17 11:50:25.769134767~2019-05-17 12:05:34.151631198.txt': 0,\n",
       " './graph_5_17/2019-05-17 09:01:32.730750122~2019-05-17 09:16:44.320659825.txt': 0,\n",
       " './graph_5_17/2019-05-17 13:06:40.331492736~2019-05-17 13:21:54.750460761.txt': 0,\n",
       " './graph_5_17/2019-05-17 00:45:36.772457892~2019-05-17 01:00:43.031312698.txt': 0,\n",
       " './graph_5_17/2019-05-17 11:35:22.890857355~2019-05-17 11:50:25.769134767.txt': 0,\n",
       " './graph_5_17/2019-05-17 06:40:14.072014912~2019-05-17 06:55:30.760988446.txt': 0,\n",
       " './graph_5_17/2019-05-17 03:04:02.082047814~2019-05-17 03:19:06.352094009.txt': 0,\n",
       " './graph_5_17/2019-05-17 05:53:46.081922263~2019-05-17 06:09:26.061933299.txt': 0,\n",
       " './graph_5_17/2019-05-17 00:00:00.011404169~2019-05-17 00:15:19.121412431.txt': 0,\n",
       " './graph_5_17/2019-05-17 07:27:34.071942527~2019-05-17 07:44:04.071921360.txt': 0,\n",
       " './graph_5_17/2019-05-17 10:02:11.321524261~2019-05-17 10:17:26.881636687.txt': 0,\n",
       " './graph_5_17/2019-05-17 16:39:33.741234335~2019-05-17 16:54:49.968820600.txt': 0,\n",
       " './graph_5_17/2019-05-17 07:59:22.051873504~2019-05-17 08:14:56.081692880.txt': 0,\n",
       " './graph_5_17/2019-05-17 12:20:48.219175447~2019-05-17 12:36:20.051747107.txt': 0,\n",
       " './graph_5_17/2019-05-17 09:31:47.131585900~2019-05-17 09:46:50.071747334.txt': 0,\n",
       " './graph_5_17/2019-05-17 02:33:34.082266094~2019-05-17 02:48:48.272633335.txt': 0,\n",
       " './graph_5_17/2019-05-17 03:50:24.232420260~2019-05-17 04:05:38.062092796.txt': 0,\n",
       " './graph_5_17/2019-05-17 17:09:59.490279010~2019-05-17 17:26:04.070998589.txt': 0,\n",
       " './graph_5_17/2019-05-17 08:45:38.051874367~2019-05-17 09:01:32.730750122.txt': 0,\n",
       " './graph_5_17/2019-05-17 03:35:22.072176911~2019-05-17 03:50:24.232420260.txt': 0,\n",
       " './graph_5_17/2019-05-17 08:14:56.081692880~2019-05-17 08:30:22.081640414.txt': 0,\n",
       " './graph_5_17/2019-05-17 02:17:12.251503432~2019-05-17 02:33:34.082266094.txt': 0,\n",
       " './graph_5_17/2019-05-17 13:52:20.000650535~2019-05-17 14:07:34.051622439.txt': 0,\n",
       " './graph_5_17/2019-05-17 09:46:50.071747334~2019-05-17 10:02:11.321524261.txt': 0,\n",
       " './graph_5_17/2019-05-17 08:30:22.081640414~2019-05-17 08:45:38.051874367.txt': 0,\n",
       " './graph_5_17/2019-05-17 00:30:22.942273426~2019-05-17 00:45:36.772457892.txt': 0,\n",
       " './graph_5_17/2019-05-17 14:52:40.948619952~2019-05-17 15:07:56.121477590.txt': 0,\n",
       " './graph_5_17/2019-05-17 13:37:08.741478317~2019-05-17 13:52:20.000650535.txt': 0,\n",
       " './graph_5_17/2019-05-17 16:24:30.611175860~2019-05-17 16:39:33.741234335.txt': 0,\n",
       " './graph_5_17/2019-05-17 12:05:34.151631198~2019-05-17 12:20:48.219175447.txt': 0,\n",
       " './graph_5_17/2019-05-17 12:51:28.051740428~2019-05-17 13:06:40.331492736.txt': 0,\n",
       " './graph_5_17/2019-05-17 04:36:14.272391092~2019-05-17 04:51:18.351942306.txt': 0,\n",
       " './graph_5_17/2019-05-17 15:07:56.121477590~2019-05-17 15:23:16.091314858.txt': 0,\n",
       " './graph_5_17/2019-05-17 07:11:16.051875332~2019-05-17 07:27:34.071942527.txt': 0,\n",
       " './graph_5_17/2019-05-17 17:26:04.070998589~2019-05-17 17:42:18.071078452.txt': 0,\n",
       " './graph_5_17/2019-05-17 11:04:49.619210641~2019-05-17 11:19:49.740854239.txt': 0}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Anomaly Detection 5-15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "index_count: 0\n",
      "thr: 2.194360056990856\n",
      "./graph_5_15/2019-05-15 00:00:00.006192320~2019-05-15 00:15:13.743767966.txt    3.7553629815613583  count: 8485  percentage: 0.08631388346354167  node count: 493  edge count: 451\n",
      "index_count: 1\n",
      "thr: 1.8967011079816516\n",
      "./graph_5_15/2019-05-15 00:15:13.743767966~2019-05-15 00:30:15.346246304.txt    3.3338512660766  count: 5351  percentage: 0.08428364415322581  node count: 204  edge count: 197\n",
      "index_count: 2\n",
      "thr: 2.0548762933032765\n",
      "./graph_5_15/2019-05-15 00:30:15.346246304~2019-05-15 00:45:38.935375235.txt    3.604907765926862  count: 9043  percentage: 0.08176902488425926  node count: 222  edge count: 222\n",
      "index_count: 3\n",
      "thr: 1.5347027361432513\n",
      "./graph_5_15/2019-05-15 00:45:38.935375235~2019-05-15 01:00:49.986348448.txt    3.4026851467634915  count: 3159  percentage: 0.051416015625  node count: 148  edge count: 144\n",
      "index_count: 4\n",
      "thr: 1.811901356497414\n",
      "./graph_5_15/2019-05-15 01:00:49.986348448~2019-05-15 01:16:13.516335500.txt    3.4977859843649006  count: 6851  percentage: 0.0662418780940594  node count: 233  edge count: 222\n",
      "index_count: 5\n",
      "thr: 1.8509196640360028\n",
      "./graph_5_15/2019-05-15 01:16:13.516335500~2019-05-15 01:31:15.876102508.txt    3.850737416453858  count: 6117  percentage: 0.06158384342783505  node count: 182  edge count: 179\n",
      "index_count: 6\n",
      "thr: 1.8035085562501862\n",
      "./graph_5_15/2019-05-15 01:31:15.876102508~2019-05-15 01:46:35.756168353.txt    4.285655194886221  count: 2823  percentage: 0.04672603283898305  node count: 104  edge count: 100\n",
      "index_count: 7\n",
      "thr: 2.3240332468109024\n",
      "./graph_5_15/2019-05-15 01:46:35.756168353~2019-05-15 02:02:03.445171413.txt    4.291559206658279  count: 3510  percentage: 0.0714111328125  node count: 120  edge count: 116\n",
      "index_count: 8\n",
      "thr: 2.2472062890193123\n",
      "./graph_5_15/2019-05-15 02:02:03.445171413~2019-05-15 02:17:36.366097118.txt    3.814528606741681  count: 9896  percentage: 0.09031834112149532  node count: 214  edge count: 206\n",
      "index_count: 9\n",
      "thr: 2.4824777400649376\n",
      "./graph_5_15/2019-05-15 02:17:36.366097118~2019-05-15 02:33:08.316023927.txt    4.713608911401915  count: 1969  percentage: 0.06867327008928571  node count: 144  edge count: 138\n",
      "index_count: 10\n",
      "thr: 1.3496595813324825\n",
      "./graph_5_15/2019-05-15 02:33:08.316023927~2019-05-15 02:48:12.346047654.txt    2.809285390238981  count: 5526  percentage: 0.062028556034482756  node count: 176  edge count: 171\n",
      "index_count: 11\n",
      "thr: 1.7251529545395967\n",
      "./graph_5_15/2019-05-15 02:48:12.346047654~2019-05-15 03:04:19.506191624.txt    3.351530875971443  count: 1900  percentage: 0.061848958333333336  node count: 77  edge count: 75\n",
      "index_count: 12\n",
      "thr: 1.5387689812757113\n",
      "./graph_5_15/2019-05-15 03:04:19.506191624~2019-05-15 03:19:20.336382785.txt    3.1983980732761887  count: 4172  percentage: 0.05991498161764706  node count: 143  edge count: 135\n",
      "index_count: 13\n",
      "thr: 1.4959896045853929\n",
      "./graph_5_15/2019-05-15 03:19:20.336382785~2019-05-15 03:34:24.565949954.txt    3.283423827696768  count: 3002  percentage: 0.05054552801724138  node count: 76  edge count: 69\n",
      "index_count: 14\n",
      "thr: 0.913555049450676\n",
      "./graph_5_15/2019-05-15 03:34:24.565949954~2019-05-15 03:50:08.335957105.txt    2.6646941949314913  count: 571  percentage: 0.030978732638888888  node count: 17  edge count: 14\n",
      "index_count: 15\n",
      "thr: 0.4304530374863928\n",
      "./graph_5_15/2019-05-15 03:50:08.335957105~2019-05-15 04:07:57.715926628.txt    1.5614719877117558  count: 95  percentage: 0.015462239583333334  node count: 21  edge count: 17\n",
      "index_count: 16\n",
      "thr: 0.3960624458373503\n",
      "./graph_5_15/2019-05-15 04:07:57.715926628~2019-05-15 04:25:32.335891238.txt    1.6448182935838576  count: 77  percentage: 0.012532552083333334  node count: 13  edge count: 10\n",
      "index_count: 17\n",
      "thr: 0.36821082758065754\n",
      "./graph_5_15/2019-05-15 04:25:32.335891238~2019-05-15 04:41:16.315832364.txt    1.628198425831466  count: 58  percentage: 0.011328125  node count: 15  edge count: 11\n",
      "index_count: 18\n",
      "thr: 0.34358954870914304\n",
      "./graph_5_15/2019-05-15 04:41:16.315832364~2019-05-15 04:56:36.326111458.txt    1.7253120766610515  count: 49  percentage: 0.0095703125  node count: 13  edge count: 11\n",
      "index_count: 19\n",
      "thr: 0.33036310206909597\n",
      "./graph_5_15/2019-05-15 04:56:36.326111458~2019-05-15 05:13:08.315787838.txt    1.4607018177777948  count: 71  percentage: 0.011555989583333334  node count: 19  edge count: 16\n",
      "index_count: 20\n",
      "thr: 0.3341501849856774\n",
      "./graph_5_15/2019-05-15 05:13:08.315787838~2019-05-15 05:28:36.315778004.txt    1.557166810129203  count: 51  percentage: 0.0099609375  node count: 13  edge count: 10\n",
      "index_count: 21\n",
      "thr: 0.301371222064687\n",
      "./graph_5_15/2019-05-15 05:28:36.315778004~2019-05-15 05:44:40.326005052.txt    1.5914776348150694  count: 39  percentage: 0.0076171875  node count: 11  edge count: 9\n",
      "index_count: 22\n",
      "thr: 0.7936930945708711\n",
      "./graph_5_15/2019-05-15 05:44:40.326005052~2019-05-15 06:00:28.326014326.txt    2.9487243729479173  count: 85  percentage: 0.0166015625  node count: 43  edge count: 40\n",
      "index_count: 23\n",
      "thr: 0.3626637356295144\n",
      "./graph_5_15/2019-05-15 06:00:28.326014326~2019-05-15 06:17:32.325991084.txt    1.6025852687431104  count: 66  percentage: 0.0107421875  node count: 15  edge count: 13\n",
      "index_count: 24\n",
      "thr: 0.2923955686673466\n",
      "./graph_5_15/2019-05-15 06:17:32.325991084~2019-05-15 06:33:44.315665234.txt    1.7324440265074372  count: 32  percentage: 0.00625  node count: 11  edge count: 9\n",
      "index_count: 25\n",
      "thr: 0.33483349078738006\n",
      "./graph_5_15/2019-05-15 06:33:44.315665234~2019-05-15 06:49:36.336078067.txt    1.4102531912034018  count: 57  percentage: 0.0111328125  node count: 13  edge count: 10\n",
      "index_count: 26\n",
      "thr: 0.39967149238490884\n",
      "./graph_5_15/2019-05-15 06:49:36.336078067~2019-05-15 07:05:22.335601187.txt    1.511063497979194  count: 64  percentage: 0.0125  node count: 22  edge count: 19\n",
      "index_count: 27\n",
      "thr: 0.33274821449447495\n",
      "./graph_5_15/2019-05-15 07:05:22.335601187~2019-05-15 07:23:00.336037899.txt    1.8811491966247558  count: 35  percentage: 0.005696614583333333  node count: 11  edge count: 9\n",
      "index_count: 28\n",
      "thr: 0.3140180714352244\n",
      "./graph_5_15/2019-05-15 07:23:00.336037899~2019-05-15 07:39:16.336018368.txt    2.2635054815383184  count: 21  percentage: 0.0041015625  node count: 5  edge count: 5\n",
      "index_count: 29\n",
      "thr: 0.3683737276369214\n",
      "./graph_5_15/2019-05-15 07:39:16.336018368~2019-05-15 07:55:36.325862117.txt    2.1499398350715637  count: 32  percentage: 0.00625  node count: 11  edge count: 9\n",
      "index_count: 30\n",
      "thr: 0.33508835884546134\n",
      "./graph_5_15/2019-05-15 07:55:36.325862117~2019-05-15 08:12:44.335538340.txt    1.5153675457196576  count: 56  percentage: 0.009114583333333334  node count: 17  edge count: 15\n",
      "index_count: 31\n",
      "thr: 0.2989161524500776\n",
      "./graph_5_15/2019-05-15 08:12:44.335538340~2019-05-15 08:29:08.335580692.txt    1.944444910933574  count: 24  percentage: 0.0046875  node count: 9  edge count: 8\n",
      "index_count: 32\n",
      "thr: 2.227493341714963\n",
      "./graph_5_15/2019-05-15 08:29:08.335580692~2019-05-15 08:46:56.325786951.txt    4.10687940284269  count: 481  percentage: 0.07828776041666667  node count: 121  edge count: 120\n",
      "index_count: 33\n",
      "thr: 1.8092681565550763\n",
      "./graph_5_15/2019-05-15 08:46:56.325786951~2019-05-15 09:02:05.234725465.txt    3.688933145800395  count: 478  percentage: 0.06668526785714286  node count: 11  edge count: 9\n",
      "index_count: 34\n",
      "thr: 1.7845875580414585\n",
      "./graph_5_15/2019-05-15 09:02:05.234725465~2019-05-15 09:17:07.225888229.txt    3.986625826737591  count: 5350  percentage: 0.05224609375  node count: 385  edge count: 382\n",
      "index_count: 35\n",
      "thr: 1.4541184346809772\n",
      "./graph_5_15/2019-05-15 09:17:07.225888229~2019-05-15 09:32:11.065706817.txt    2.712613080968529  count: 4157  percentage: 0.07122053179824561  node count: 113  edge count: 103\n",
      "index_count: 36\n",
      "thr: 1.960587073417376\n",
      "./graph_5_15/2019-05-15 09:32:11.065706817~2019-05-15 09:48:38.684621579.txt    3.8664344308455894  count: 6621  percentage: 0.067352294921875  node count: 107  edge count: 106\n",
      "index_count: 37\n",
      "thr: 2.4424937242645512\n",
      "./graph_5_15/2019-05-15 09:48:38.684621579~2019-05-15 10:03:41.465554650.txt    4.182358261437444  count: 5472  percentage: 0.08349609375  node count: 154  edge count: 144\n",
      "index_count: 38\n",
      "thr: 1.363769493459408\n",
      "./graph_5_15/2019-05-15 10:03:41.465554650~2019-05-15 10:18:48.315349922.txt    2.4541032879242217  count: 5423  percentage: 0.07675215126811594  node count: 174  edge count: 176\n",
      "index_count: 39\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "thr: 2.4039723503206485\n",
      "./graph_5_15/2019-05-15 10:18:48.315349922~2019-05-15 10:34:18.902982618.txt    4.318013438905655  count: 6537  percentage: 0.08290635146103896  node count: 179  edge count: 172\n",
      "index_count: 40\n",
      "thr: 1.9027204455547244\n",
      "./graph_5_15/2019-05-15 10:34:18.902982618~2019-05-15 10:49:19.662777306.txt    3.6041473408064464  count: 3676  percentage: 0.06526988636363637  node count: 140  edge count: 134\n",
      "index_count: 41\n",
      "thr: 2.3233825606719876\n",
      "./graph_5_15/2019-05-15 10:49:19.662777306~2019-05-15 11:04:34.935316812.txt    4.2313182064819435  count: 7594  percentage: 0.07490924873737374  node count: 124  edge count: 119\n",
      "index_count: 42\n",
      "thr: 1.8074586093199594\n",
      "./graph_5_15/2019-05-15 11:04:34.935316812~2019-05-15 11:20:14.345345440.txt    3.140305711104982  count: 6007  percentage: 0.08262268926056338  node count: 147  edge count: 140\n",
      "index_count: 43\n",
      "thr: 1.8356234974370471\n",
      "./graph_5_15/2019-05-15 11:20:14.345345440~2019-05-15 11:35:14.525271329.txt    3.2895649384241703  count: 9099  percentage: 0.07863488661504425  node count: 208  edge count: 204\n",
      "index_count: 44\n",
      "thr: 1.667320468854776\n",
      "./graph_5_15/2019-05-15 11:35:14.525271329~2019-05-15 11:51:01.435516598.txt    3.3657420873641968  count: 5874  percentage: 0.064453125  node count: 153  edge count: 144\n",
      "index_count: 45\n",
      "thr: 2.674420505401372\n",
      "./graph_5_15/2019-05-15 11:51:01.435516598~2019-05-15 12:07:06.345256228.txt    4.579094775664874  count: 9293  percentage: 0.08810869235436893  node count: 244  edge count: 237\n",
      "index_count: 46\n",
      "thr: 2.0111331853522847\n",
      "./graph_5_15/2019-05-15 12:07:06.345256228~2019-05-15 12:22:23.514541754.txt    3.765099536738383  count: 8872  percentage: 0.07405181623931624  node count: 260  edge count: 251\n",
      "index_count: 47\n",
      "thr: 1.644810914018587\n",
      "./graph_5_15/2019-05-15 12:22:23.514541754~2019-05-15 12:38:28.335591405.txt    3.786204528877948  count: 4469  percentage: 0.04959383877840909  node count: 116  edge count: 113\n",
      "index_count: 48\n",
      "thr: 1.823623944078935\n",
      "./graph_5_15/2019-05-15 12:38:28.335591405~2019-05-15 12:53:47.114274288.txt    3.342645482224346  count: 3772  percentage: 0.06950176886792453  node count: 89  edge count: 91\n",
      "index_count: 49\n",
      "thr: 3.0326054817550947\n",
      "./graph_5_15/2019-05-15 12:53:47.114274288~2019-05-15 13:08:48.445537909.txt    4.467972467329993  count: 16023  percentage: 0.1125716614208633  node count: 355  edge count: 351\n",
      "index_count: 50\n",
      "thr: 1.8872154934627623\n",
      "./graph_5_15/2019-05-15 13:08:48.445537909~2019-05-15 13:23:52.255450878.txt    3.593101685591299  count: 3882  percentage: 0.06214779713114754  node count: 155  edge count: 149\n",
      "index_count: 51\n",
      "thr: 2.441397819762378\n",
      "./graph_5_15/2019-05-15 13:23:52.255450878~2019-05-15 13:39:04.835250858.txt    3.991797171726128  count: 6369  percentage: 0.0971832275390625  node count: 229  edge count: 222\n",
      "index_count: 52\n",
      "thr: 1.5312749192812198\n",
      "./graph_5_15/2019-05-15 13:39:04.835250858~2019-05-15 13:54:20.623015847.txt    3.3051855231980714  count: 5233  percentage: 0.05678168402777778  node count: 210  edge count: 220\n",
      "index_count: 53\n",
      "thr: 1.9505282717632575\n",
      "./graph_5_15/2019-05-15 13:54:20.623015847~2019-05-15 14:09:38.985029953.txt    3.7896994965042126  count: 4724  percentage: 0.06407335069444445  node count: 146  edge count: 139\n",
      "index_count: 54\n",
      "thr: 2.1017754445918215\n",
      "./graph_5_15/2019-05-15 14:09:38.985029953~2019-05-15 14:25:00.732554067.txt    4.486710976142493  count: 4492  percentage: 0.056239983974358976  node count: 170  edge count: 165\n",
      "index_count: 55\n",
      "thr: 2.137983343857202\n",
      "./graph_5_15/2019-05-15 14:25:00.732554067~2019-05-15 14:40:58.314993755.txt    3.7814219838654073  count: 5584  percentage: 0.08655753968253968  node count: 194  edge count: 187\n",
      "index_count: 56\n",
      "thr: 2.487790690556732\n",
      "./graph_5_15/2019-05-15 14:40:58.314993755~2019-05-15 14:56:22.325270164.txt    3.9621975956652475  count: 11289  percentage: 0.10400390625  node count: 315  edge count: 309\n",
      "index_count: 57\n",
      "thr: 2.0454429060994768\n",
      "./graph_5_15/2019-05-15 14:56:22.325270164~2019-05-15 15:11:29.355153391.txt    3.6953639099055073  count: 9245  percentage: 0.08598400297619048  node count: 212  edge count: 205\n",
      "index_count: 58\n",
      "thr: 2.4517000504435944\n",
      "./graph_5_15/2019-05-15 15:11:29.355153391~2019-05-15 15:26:30.315069987.txt    3.9225086803424265  count: 9526  percentage: 0.0959044780927835  node count: 224  edge count: 214\n",
      "index_count: 59\n",
      "thr: 2.009576984653087\n",
      "./graph_5_15/2019-05-15 15:26:30.315069987~2019-05-15 15:41:58.325198150.txt    3.400477163307488  count: 3122  percentage: 0.0846896701388889  node count: 195  edge count: 187\n",
      "index_count: 60\n",
      "thr: 1.1329268239092265\n",
      "./graph_5_15/2019-05-15 15:41:58.325198150~2019-05-15 15:56:59.265333194.txt    3.384459689455229  count: 1066  percentage: 0.026692708333333332  node count: 139  edge count: 134\n",
      "index_count: 61\n",
      "thr: 1.5381453226093362\n",
      "./graph_5_15/2019-05-15 15:56:59.265333194~2019-05-15 16:12:04.095060867.txt    3.396712640625583  count: 3494  percentage: 0.05503402217741935  node count: 150  edge count: 142\n",
      "index_count: 62\n",
      "thr: 1.481228620112359\n",
      "./graph_5_15/2019-05-15 16:12:04.095060867~2019-05-15 16:27:36.075256581.txt    3.0438914332760323  count: 6358  percentage: 0.06335698341836735  node count: 179  edge count: 175\n",
      "index_count: 63\n",
      "thr: 2.385700667317554\n",
      "./graph_5_15/2019-05-15 16:27:36.075256581~2019-05-15 16:43:24.282499513.txt    4.249138731724248  count: 5359  percentage: 0.08440965221774194  node count: 254  edge count: 246\n",
      "index_count: 64\n",
      "thr: 2.007373430944597\n",
      "./graph_5_15/2019-05-15 16:43:24.282499513~2019-05-15 16:58:34.225243276.txt    3.2706494330501314  count: 2777  percentage: 0.09685407366071429  node count: 77  edge count: 70\n",
      "index_count: 65\n",
      "thr: 2.262651709880603\n",
      "./graph_5_15/2019-05-15 16:58:34.225243276~2019-05-15 17:13:45.412245202.txt    3.5678735043228893  count: 6632  percentage: 0.10617315573770492  node count: 188  edge count: 187\n",
      "index_count: 66\n",
      "thr: 2.0934627341708616\n",
      "./graph_5_15/2019-05-15 17:13:45.412245202~2019-05-15 17:28:46.064963493.txt    4.014717055954744  count: 3574  percentage: 0.0684359681372549  node count: 140  edge count: 135\n",
      "index_count: 67\n",
      "thr: 2.2879828388153767\n",
      "./graph_5_15/2019-05-15 17:28:46.064963493~2019-05-15 17:44:50.054854387.txt    3.881899551710031  count: 8272  percentage: 0.09076544943820225  node count: 211  edge count: 203\n",
      "index_count: 68\n",
      "thr: 2.122992949527118\n",
      "./graph_5_15/2019-05-15 17:44:50.054854387~2019-05-15 17:59:50.764934674.txt    3.3878687674718333  count: 5135  percentage: 0.10669464760638298  node count: 124  edge count: 116\n",
      "index_count: 69\n",
      "thr: 2.1704354385289384\n",
      "./graph_5_15/2019-05-15 17:59:50.764934674~2019-05-15 18:15:04.084749724.txt    3.7313468410509048  count: 5575  percentage: 0.08641803075396826  node count: 204  edge count: 196\n",
      "index_count: 70\n",
      "thr: 2.2606844288044603\n",
      "./graph_5_15/2019-05-15 18:15:04.084749724~2019-05-15 18:30:12.692563554.txt    4.068686050919631  count: 4003  percentage: 0.07977917729591837  node count: 140  edge count: 132\n",
      "index_count: 71\n",
      "thr: 1.8899822956883177\n",
      "./graph_5_15/2019-05-15 18:30:12.692563554~2019-05-15 18:45:36.312255908.txt    3.432346852684542  count: 7687  percentage: 0.07985995678191489  node count: 250  edge count: 247\n",
      "index_count: 72\n",
      "thr: 2.3820443674834735\n",
      "./graph_5_15/2019-05-15 18:45:36.312255908~2019-05-15 19:00:43.932284310.txt    3.952277973755909  count: 7779  percentage: 0.09378616898148148  node count: 156  edge count: 150\n",
      "index_count: 73\n",
      "thr: 1.8234524597759507\n",
      "./graph_5_15/2019-05-15 19:00:43.932284310~2019-05-15 19:16:40.834769346.txt    3.1425319566499375  count: 8307  percentage: 0.08363200708762887  node count: 274  edge count: 270\n",
      "index_count: 74\n",
      "thr: 2.052757007725256\n",
      "./graph_5_15/2019-05-15 19:16:40.834769346~2019-05-15 19:32:22.054614132.txt    3.8366486431091373  count: 8393  percentage: 0.0738404420045045  node count: 270  edge count: 260\n",
      "index_count: 75\n",
      "thr: 2.9339508876767844\n",
      "./graph_5_15/2019-05-15 19:32:22.054614132~2019-05-15 19:47:26.074698234.txt    4.984092596856779  count: 6898  percentage: 0.09906364889705882  node count: 123  edge count: 119\n",
      "index_count: 76\n",
      "thr: 2.3316813689751563\n",
      "./graph_5_15/2019-05-15 19:47:26.074698234~2019-05-15 20:02:51.452177789.txt    4.53277850042124  count: 4596  percentage: 0.0669892723880597  node count: 174  edge count: 167\n",
      "index_count: 77\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "thr: 2.1255446488607967\n",
      "./graph_5_15/2019-05-15 20:02:51.452177789~2019-05-15 20:18:20.054640718.txt    3.9580907465360844  count: 2235  percentage: 0.0704070060483871  node count: 84  edge count: 79\n",
      "index_count: 78\n",
      "thr: 1.636668804430908\n",
      "./graph_5_15/2019-05-15 20:18:20.054640718~2019-05-15 20:33:20.173943586.txt    3.5333224417396347  count: 4349  percentage: 0.057392842060810814  node count: 166  edge count: 156\n",
      "index_count: 79\n",
      "thr: 2.823731389306513\n",
      "./graph_5_15/2019-05-15 20:33:20.173943586~2019-05-15 20:48:26.344519703.txt    4.345388863231559  count: 7176  percentage: 0.10617897727272728  node count: 154  edge count: 147\n",
      "index_count: 80\n",
      "thr: 2.35814581618105\n",
      "./graph_5_15/2019-05-15 20:48:26.344519703~2019-05-15 21:03:34.932076311.txt    4.466750115694744  count: 6917  percentage: 0.07263314852150538  node count: 249  edge count: 245\n",
      "index_count: 81\n",
      "thr: 1.7247192530974131\n",
      "./graph_5_15/2019-05-15 21:03:34.932076311~2019-05-15 21:18:36.594847325.txt    3.2876171714520477  count: 4684  percentage: 0.0626605308219178  node count: 144  edge count: 144\n",
      "index_count: 82\n",
      "thr: 1.646194470475106\n",
      "./graph_5_15/2019-05-15 21:18:36.594847325~2019-05-15 21:33:37.744873139.txt    3.004021282828775  count: 5102  percentage: 0.07549124053030302  node count: 252  edge count: 246\n",
      "index_count: 83\n",
      "thr: 2.429835800002369\n",
      "./graph_5_15/2019-05-15 21:33:37.744873139~2019-05-15 21:48:38.194423026.txt    4.101023526784448  count: 11705  percentage: 0.09446829803719008  node count: 260  edge count: 250\n",
      "index_count: 84\n",
      "thr: 2.091556213172371\n",
      "./graph_5_15/2019-05-15 21:48:38.194423026~2019-05-15 22:03:43.404738808.txt    3.7331281998096957  count: 7285  percentage: 0.08084383877840909  node count: 187  edge count: 177\n",
      "index_count: 85\n",
      "thr: 2.6192213068467907\n",
      "./graph_5_15/2019-05-15 22:03:43.404738808~2019-05-15 22:19:00.063809691.txt    4.280797997896539  count: 5387  percentage: 0.0922937225877193  node count: 172  edge count: 164\n",
      "index_count: 86\n",
      "thr: 1.9600708462386034\n",
      "./graph_5_15/2019-05-15 22:19:00.063809691~2019-05-15 22:34:04.714368495.txt    3.8672301911501314  count: 6386  percentage: 0.06564555921052631  node count: 183  edge count: 176\n",
      "index_count: 87\n",
      "thr: 1.7857537569783382\n",
      "./graph_5_15/2019-05-15 22:34:04.714368495~2019-05-15 22:49:14.843539280.txt    3.5003883049712847  count: 3776  percentage: 0.06828703703703703  node count: 77  edge count: 71\n",
      "index_count: 88\n",
      "thr: 2.630418994418645\n",
      "./graph_5_15/2019-05-15 22:49:14.843539280~2019-05-15 23:04:18.584315206.txt    4.456689482859799  count: 5826  percentage: 0.08620383522727272  node count: 155  edge count: 128\n",
      "index_count: 89\n",
      "thr: 2.4603890209254207\n",
      "./graph_5_15/2019-05-15 23:04:18.584315206~2019-05-15 23:19:21.464426330.txt    5.347500517444977  count: 3701  percentage: 0.058294480846774195  node count: 117  edge count: 113\n",
      "index_count: 90\n",
      "thr: 2.3481305892453763\n",
      "./graph_5_15/2019-05-15 23:19:21.464426330~2019-05-15 23:34:24.683386847.txt    4.154656218765412  count: 9244  percentage: 0.07849864130434783  node count: 258  edge count: 253\n",
      "index_count: 91\n",
      "thr: 1.7130346986920997\n",
      "./graph_5_15/2019-05-15 23:34:24.683386847~2019-05-15 23:49:33.091873325.txt    3.908265923658519  count: 4887  percentage: 0.051874575407608696  node count: 266  edge count: 261\n"
     ]
    }
   ],
   "source": [
    "# node_IDF=torch.load(\"node_IDF_5_15\")\n",
    "node_IDF=torch.load(\"node_IDF\")\n",
    "y_data_5_15=[]\n",
    "df_list_5_15=[]\n",
    "# node_set_list=[]\n",
    "history_list_5_15=[]\n",
    "tw_que=[]\n",
    "his_tw={}\n",
    "current_tw={}\n",
    "loss_list_5_15=[]\n",
    "\n",
    "file_path_list=[]\n",
    "\n",
    "file_path=\"./graph_5_15/\"\n",
    "file_l=os.listdir(\"./graph_5_15/\")\n",
    "for i in file_l:\n",
    "    file_path_list.append(file_path+i)\n",
    "\n",
    "index_count=0\n",
    "for f_path in sorted(file_path_list):\n",
    "    f=open(f_path)\n",
    "    edge_loss_list=[]\n",
    "    edge_list=[]\n",
    "    print('index_count:',index_count)\n",
    "    \n",
    "    # Figure out which nodes are anomalous in this time window\n",
    "    for line in f:\n",
    "        l=line.strip()\n",
    "        jdata=eval(l)\n",
    "        edge_loss_list.append(jdata['loss'])\n",
    "        edge_list.append([str(jdata['srcmsg']),str(jdata['dstmsg'])])\n",
    "    df_list_5_15.append(pd.DataFrame(edge_loss_list))\n",
    "    count,loss_avg,node_set,edge_set=cal_anomaly_loss(edge_loss_list,edge_list,\"./graph_5_15/\")\n",
    "\n",
    "    current_tw['name']=f_path\n",
    "    current_tw['loss']=loss_avg\n",
    "    current_tw['index']=index_count\n",
    "    current_tw['nodeset']=node_set\n",
    "\n",
    "    # Incrementally construct the queues\n",
    "    added_que_flag=False\n",
    "    for hq in history_list_5_15:\n",
    "        for his_tw in hq:\n",
    "            if cal_set_rel_bak(current_tw['nodeset'],his_tw['nodeset'],file_list)!=0 and current_tw['name']!=his_tw['name']:\n",
    "                print(\"history queue:\",his_tw['name'])\n",
    "                # check if there are intersection between two time windows.\n",
    "                hq.append(copy.deepcopy(current_tw))\n",
    "                added_que_flag=True\n",
    "                break\n",
    "            if added_que_flag:\n",
    "                break\n",
    "    if added_que_flag is False:\n",
    "        temp_hq=[copy.deepcopy(current_tw)]\n",
    "        history_list_5_15.append(temp_hq)\n",
    "    index_count+=1\n",
    "    loss_list_5_15.append(loss_avg)\n",
    "    print( f_path,\"  \",loss_avg,\" count:\",count,\" percentage:\",count/len(edge_list),\" node count:\",len(node_set),\" edge count:\",len(edge_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "name_list=[]\n",
    "for hl in history_list_5_15:\n",
    "    loss_count=0\n",
    "    for hq in hl:\n",
    "        if loss_count==0:\n",
    "            loss_count=(loss_count+1)*(hq['loss']+1)\n",
    "        else:\n",
    "            loss_count=(loss_count)*(hq['loss']+1)\n",
    "#     name_list=[]\n",
    "    if loss_count>100:\n",
    "        name_list=[]\n",
    "        for i in hl:\n",
    "            name_list.append(i['name']) \n",
    "        print(name_list)\n",
    "        for i in name_list:\n",
    "            pred_label[i]=1\n",
    "        print(loss_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Anoamly Detection 5-16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "index_count: 0\n",
      "thr: 2.725104913739848\n",
      "./graph_5_16/2019-05-16 00:00:00.014304910~2019-05-16 00:15:02.554575844.txt    4.585362636863654  count: 3257  percentage: 0.08596389358108109  node count: 189  edge count: 145\n",
      "index_count: 1\n",
      "thr: 2.242179023565214\n",
      "./graph_5_16/2019-05-16 00:15:02.554575844~2019-05-16 00:30:19.324454842.txt    4.432748329445932  count: 4066  percentage: 0.05754642210144927  node count: 213  edge count: 262\n",
      "index_count: 2\n",
      "thr: 2.3234012205822383\n",
      "./graph_5_16/2019-05-16 00:30:19.324454842~2019-05-16 00:45:31.114609140.txt    3.9136210504051414  count: 4756  percentage: 0.08444602272727272  node count: 141  edge count: 134\n",
      "index_count: 3\n",
      "thr: 1.6817430690895092\n",
      "./graph_5_16/2019-05-16 00:45:31.114609140~2019-05-16 01:01:06.524611321.txt    3.068151815172695  count: 7839  percentage: 0.07579478650990099  node count: 249  edge count: 242\n",
      "index_count: 4\n",
      "thr: 1.5668066564863836\n",
      "./graph_5_16/2019-05-16 01:01:06.524611321~2019-05-16 01:17:54.074506572.txt    3.120767466474378  count: 2090  percentage: 0.06002987132352941  node count: 262  edge count: 256\n",
      "index_count: 5\n",
      "thr: 1.4377303558071752\n",
      "./graph_5_16/2019-05-16 01:17:54.074506572~2019-05-16 01:32:54.901798763.txt    3.26773539864059  count: 896  percentage: 0.03977272727272727  node count: 199  edge count: 193\n",
      "index_count: 6\n",
      "thr: 1.781611132982948\n",
      "./graph_5_16/2019-05-16 01:32:54.901798763~2019-05-16 01:48:08.801926929.txt    4.042876928667479  count: 1422  percentage: 0.04339599609375  node count: 262  edge count: 252\n",
      "index_count: 7\n",
      "thr: 1.7995970558266745\n",
      "./graph_5_16/2019-05-16 01:48:08.801926929~2019-05-16 02:03:13.643316639.txt    4.607849374613357  count: 671  percentage: 0.03448807565789474  node count: 95  edge count: 87\n",
      "index_count: 8\n",
      "thr: 1.774785220392198\n",
      "./graph_5_16/2019-05-16 02:03:13.643316639~2019-05-16 02:19:39.961526318.txt    3.8147364642150707  count: 1407  percentage: 0.049072265625  node count: 158  edge count: 150\n",
      "index_count: 9\n",
      "thr: 1.831362086199362\n",
      "./graph_5_16/2019-05-16 02:19:39.961526318~2019-05-16 02:35:30.054106322.txt    3.1903553091884  count: 1741  percentage: 0.07728160511363637  node count: 105  edge count: 92\n",
      "index_count: 10\n",
      "thr: 3.1187875125814837\n",
      "./graph_5_16/2019-05-16 02:35:30.054106322~2019-05-16 02:50:35.811829269.txt    4.8880030510639445  count: 3021  percentage: 0.10926649305555555  node count: 71  edge count: 63\n",
      "index_count: 11\n",
      "thr: 1.493284282032794\n",
      "./graph_5_16/2019-05-16 02:50:35.811829269~2019-05-16 03:06:05.943294327.txt    3.1955079911364916  count: 1478  percentage: 0.051548549107142856  node count: 268  edge count: 257\n",
      "index_count: 12\n",
      "thr: 2.180176180559352\n",
      "./graph_5_16/2019-05-16 03:06:05.943294327~2019-05-16 03:21:47.303111250.txt    4.3437136354900545  count: 1512  percentage: 0.06419836956521739  node count: 172  edge count: 168\n",
      "index_count: 13\n",
      "thr: 1.7947890257995285\n",
      "./graph_5_16/2019-05-16 03:21:47.303111250~2019-05-16 03:36:53.523065535.txt    3.207891808773007  count: 1348  percentage: 0.06928453947368421  node count: 112  edge count: 109\n",
      "index_count: 14\n",
      "thr: 1.4160249780475946\n",
      "./graph_5_16/2019-05-16 03:36:53.523065535~2019-05-16 03:52:06.074285645.txt    3.247045183749426  count: 630  percentage: 0.05126953125  node count: 97  edge count: 89\n",
      "index_count: 15\n",
      "thr: 1.610590957104761\n",
      "./graph_5_16/2019-05-16 03:52:06.074285645~2019-05-16 04:07:28.053987688.txt    4.181540843079851  count: 573  percentage: 0.03497314453125  node count: 91  edge count: 82\n",
      "index_count: 16\n",
      "thr: 0.42098701790416265\n",
      "./graph_5_16/2019-05-16 04:07:28.053987688~2019-05-16 04:23:52.053970386.txt    2.6456203318455  count: 44  percentage: 0.00537109375  node count: 12  edge count: 10\n",
      "index_count: 17\n",
      "thr: 0.3453743120638729\n",
      "./graph_5_16/2019-05-16 04:23:52.053970386~2019-05-16 04:39:16.074221006.txt    2.5673710266749064  count: 30  percentage: 0.004185267857142857  node count: 9  edge count: 8\n",
      "index_count: 18\n",
      "thr: 0.3405881074120019\n",
      "./graph_5_16/2019-05-16 04:39:16.074221006~2019-05-16 04:54:40.073948996.txt    1.8097107965572208  count: 51  percentage: 0.007114955357142857  node count: 11  edge count: 9\n",
      "index_count: 19\n",
      "thr: 0.3204777932355749\n",
      "./graph_5_16/2019-05-16 04:54:40.073948996~2019-05-16 05:10:12.053923923.txt    1.6195712960325181  count: 64  percentage: 0.0078125  node count: 25  edge count: 21\n",
      "index_count: 20\n",
      "thr: 0.328325711362027\n",
      "./graph_5_16/2019-05-16 05:10:12.053923923~2019-05-16 05:25:30.074155632.txt    1.7445238238343825  count: 52  percentage: 0.007254464285714286  node count: 11  edge count: 9\n",
      "index_count: 21\n",
      "thr: 0.2988384964303306\n",
      "./graph_5_16/2019-05-16 05:25:30.074155632~2019-05-16 05:41:08.053785658.txt    2.149181624253591  count: 30  percentage: 0.004185267857142857  node count: 9  edge count: 8\n",
      "index_count: 22\n",
      "thr: 0.33159613272956184\n",
      "./graph_5_16/2019-05-16 05:41:08.053785658~2019-05-16 05:56:36.063970033.txt    2.100663187198861  count: 43  percentage: 0.005998883928571429  node count: 11  edge count: 9\n",
      "index_count: 23\n",
      "thr: 0.31048403351492276\n",
      "./graph_5_16/2019-05-16 05:56:36.063970033~2019-05-16 06:12:56.073950599.txt    1.6222676671468295  count: 65  percentage: 0.0079345703125  node count: 17  edge count: 15\n",
      "index_count: 24\n",
      "thr: 0.3009354330261416\n",
      "./graph_5_16/2019-05-16 06:12:56.073950599~2019-05-16 06:28:28.074065777.txt    1.8259567101796468  count: 45  percentage: 0.006277901785714286  node count: 11  edge count: 9\n",
      "index_count: 25\n",
      "thr: 0.2690985358689162\n",
      "./graph_5_16/2019-05-16 06:28:28.074065777~2019-05-16 06:43:56.073801568.txt    1.7585679465218593  count: 38  percentage: 0.005301339285714286  node count: 11  edge count: 9\n",
      "index_count: 26\n",
      "thr: 0.302903721406667\n",
      "./graph_5_16/2019-05-16 06:43:56.073801568~2019-05-16 06:59:18.053755875.txt    1.611356532132184  count: 54  percentage: 0.007533482142857143  node count: 12  edge count: 10\n",
      "index_count: 27\n",
      "thr: 0.2813925820851324\n",
      "./graph_5_16/2019-05-16 06:59:18.053755875~2019-05-16 07:15:28.063859188.txt    1.360659750870296  count: 70  percentage: 0.008544921875  node count: 22  edge count: 19\n",
      "index_count: 28\n",
      "thr: 0.26974808732576144\n",
      "./graph_5_16/2019-05-16 07:15:28.063859188~2019-05-16 07:30:46.073981043.txt    1.3973972217904196  count: 54  percentage: 0.007533482142857143  node count: 11  edge count: 9\n",
      "index_count: 29\n",
      "thr: 0.28196921626870997\n",
      "./graph_5_16/2019-05-16 07:30:46.073981043~2019-05-16 07:45:56.083899517.txt    1.5276789584507544  count: 48  percentage: 0.006696428571428571  node count: 22  edge count: 18\n",
      "index_count: 30\n",
      "thr: 0.22320002999726246\n",
      "./graph_5_16/2019-05-16 07:45:56.083899517~2019-05-16 08:03:04.073932182.txt    1.0612171503404777  count: 72  percentage: 0.0087890625  node count: 21  edge count: 18\n",
      "index_count: 31\n",
      "thr: 0.27551396377172627\n",
      "./graph_5_16/2019-05-16 08:03:04.073932182~2019-05-16 08:19:48.053598991.txt    1.7973982718857853  count: 44  percentage: 0.00537109375  node count: 11  edge count: 9\n",
      "index_count: 32\n",
      "thr: 0.3475568481636134\n",
      "./graph_5_16/2019-05-16 08:19:48.053598991~2019-05-16 08:35:02.133692641.txt    2.2749834030866625  count: 40  percentage: 0.005580357142857143  node count: 13  edge count: 11\n",
      "index_count: 33\n",
      "thr: 0.8777323361812557\n",
      "./graph_5_16/2019-05-16 08:35:02.133692641~2019-05-16 08:50:04.103774736.txt    4.070007429403417  count: 102  percentage: 0.014229910714285714  node count: 40  edge count: 39\n",
      "index_count: 34\n",
      "thr: 1.0145026969491195\n",
      "./graph_5_16/2019-05-16 08:50:04.103774736~2019-05-16 09:05:18.093603996.txt    2.7078777237942346  count: 684  percentage: 0.0333984375  node count: 46  edge count: 42\n",
      "index_count: 35\n",
      "thr: 1.5058476989033094\n",
      "./graph_5_16/2019-05-16 09:05:18.093603996~2019-05-16 09:20:32.093582942.txt    3.32552791565188  count: 3434  percentage: 0.04790736607142857  node count: 364  edge count: 357\n",
      "index_count: 36\n",
      "thr: 2.0610995962742136\n",
      "./graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt    4.141747653654741  count: 3498  percentage: 0.05789856991525424  node count: 151  edge count: 147\n",
      "index_count: 37\n",
      "thr: 1.6287077724326542\n",
      "node: {'subject': 'nginx'}  IDF: 5.752572638825633\n",
      "history queue: ./graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt\n",
      "./graph_5_16/2019-05-16 09:36:08.903494477~2019-05-16 09:51:22.110949680.txt    3.2641165017814977  count: 2682  percentage: 0.05693783967391304  node count: 98  edge count: 93\n",
      "index_count: 38\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "thr: 2.9626802917298387\n",
      "node: {'subject': 'php-fpm'}  IDF: 5.752572638825633\n",
      "node: {'subject': 'nginx'}  IDF: 5.752572638825633\n",
      "history queue: ./graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt\n",
      "./graph_5_16/2019-05-16 09:51:22.110949680~2019-05-16 10:06:29.403713371.txt    4.688541164505954  count: 4956  percentage: 0.096796875  node count: 101  edge count: 92\n",
      "index_count: 39\n",
      "thr: 2.47509204989206\n",
      "node: {'subject': 'nginx'}  IDF: 5.752572638825633\n",
      "history queue: ./graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt\n",
      "./graph_5_16/2019-05-16 10:06:29.403713371~2019-05-16 10:21:47.983513184.txt    4.389596619022277  count: 5522  percentage: 0.08559647817460317  node count: 183  edge count: 178\n",
      "index_count: 40\n",
      "thr: 1.8960156008413767\n",
      "./graph_5_16/2019-05-16 10:21:47.983513184~2019-05-16 10:37:02.053456880.txt    3.966470134093636  count: 5141  percentage: 0.05906479779411765  node count: 179  edge count: 174\n",
      "index_count: 41\n",
      "thr: 2.7146693278060763\n",
      "./graph_5_16/2019-05-16 10:37:02.053456880~2019-05-16 10:52:13.133498417.txt    3.9904962030318676  count: 10045  percentage: 0.11962890625  node count: 129  edge count: 123\n",
      "index_count: 42\n",
      "thr: 2.32395513515784\n",
      "./graph_5_16/2019-05-16 10:52:13.133498417~2019-05-16 11:07:58.483378590.txt    4.376090318515663  count: 4218  percentage: 0.072265625  node count: 178  edge count: 176\n",
      "index_count: 43\n",
      "thr: 2.586902769712219\n",
      "./graph_5_16/2019-05-16 11:07:58.483378590~2019-05-16 11:23:00.723415561.txt    4.751203407853166  count: 6438  percentage: 0.06985677083333333  node count: 70  edge count: 62\n",
      "index_count: 44\n",
      "thr: 2.1791936737998494\n",
      "./graph_5_16/2019-05-16 11:23:00.723415561~2019-05-16 11:38:36.270922023.txt    4.15653855952581  count: 3091  percentage: 0.06707899305555555  node count: 186  edge count: 178\n",
      "index_count: 45\n",
      "thr: 1.3405620598403805\n",
      "./graph_5_16/2019-05-16 11:38:36.270922023~2019-05-16 11:54:09.383300779.txt    2.7664584484645878  count: 2832  percentage: 0.0553125  node count: 160  edge count: 154\n",
      "index_count: 46\n",
      "thr: 2.025966178822991\n",
      "./graph_5_16/2019-05-16 11:54:09.383300779~2019-05-16 12:09:10.073385814.txt    3.8875145617374525  count: 6187  percentage: 0.07192847842261904  node count: 312  edge count: 304\n",
      "index_count: 47\n",
      "thr: 2.8483238915735525\n",
      "./graph_5_16/2019-05-16 12:09:10.073385814~2019-05-16 12:24:13.482488891.txt    4.895511463474316  count: 5024  percentage: 0.08761160714285714  node count: 204  edge count: 199\n",
      "index_count: 48\n",
      "thr: 2.3431292379442334\n",
      "./graph_5_16/2019-05-16 12:24:13.482488891~2019-05-16 12:39:13.512670126.txt    4.021744091348843  count: 9055  percentage: 0.08669385723039216  node count: 178  edge count: 169\n",
      "index_count: 49\n",
      "thr: 2.274915666876548\n",
      "./graph_5_16/2019-05-16 12:39:13.512670126~2019-05-16 12:54:23.383199820.txt    3.8532706319967542  count: 7866  percentage: 0.08729137073863637  node count: 112  edge count: 102\n",
      "index_count: 50\n",
      "thr: 2.7408731811456093\n",
      "./graph_5_16/2019-05-16 12:54:23.383199820~2019-05-16 13:10:19.070779147.txt    4.219989091117329  count: 11015  percentage: 0.10650332611386139  node count: 383  edge count: 381\n",
      "index_count: 51\n",
      "thr: 2.4808546988029163\n",
      "./graph_5_16/2019-05-16 13:10:19.070779147~2019-05-16 13:25:31.470978934.txt    4.542242204591622  count: 5708  percentage: 0.07334498355263158  node count: 130  edge count: 125\n",
      "index_count: 52\n",
      "thr: 2.7861049292181286\n",
      "./graph_5_16/2019-05-16 13:25:31.470978934~2019-05-16 13:40:32.073458932.txt    4.180730466159722  count: 13364  percentage: 0.11652483258928571  node count: 360  edge count: 361\n",
      "index_count: 53\n",
      "thr: 2.747163677762293\n",
      "./graph_5_16/2019-05-16 13:40:32.073458932~2019-05-16 13:55:36.560621070.txt    4.739201164059308  count: 11262  percentage: 0.08941501524390244  node count: 290  edge count: 282\n",
      "index_count: 54\n",
      "thr: 2.2767235008134152\n",
      "./graph_5_16/2019-05-16 13:55:36.560621070~2019-05-16 14:10:44.950897261.txt    4.020838125899383  count: 8790  percentage: 0.08098098466981132  node count: 256  edge count: 254\n",
      "index_count: 55\n",
      "thr: 2.7746338911478454\n",
      "./graph_5_16/2019-05-16 14:10:44.950897261~2019-05-16 14:25:48.270690959.txt    4.198497279381716  count: 11073  percentage: 0.10922703598484848  node count: 347  edge count: 340\n",
      "index_count: 56\n",
      "thr: 1.966731013509888\n",
      "./graph_5_16/2019-05-16 14:25:48.270690959~2019-05-16 14:41:00.063228926.txt    3.997079312427534  count: 8610  percentage: 0.06780808971774194  node count: 312  edge count: 305\n",
      "index_count: 57\n",
      "thr: 2.629255718675563\n",
      "./graph_5_16/2019-05-16 14:41:00.063228926~2019-05-16 14:56:22.063206218.txt    4.316954666251937  count: 3840  percentage: 0.09615384615384616  node count: 42  edge count: 39\n",
      "index_count: 58\n",
      "thr: 3.4032532473873527\n",
      "./graph_5_16/2019-05-16 14:56:22.063206218~2019-05-16 15:11:22.163040001.txt    5.268438932273923  count: 9280  percentage: 0.10416666666666667  node count: 366  edge count: 358\n",
      "index_count: 59\n",
      "thr: 2.4144115976551843\n",
      "./graph_5_16/2019-05-16 15:11:22.163040001~2019-05-16 15:27:01.353104819.txt    3.8002382590894617  count: 8679  percentage: 0.09855332485465117  node count: 321  edge count: 320\n",
      "index_count: 60\n",
      "thr: 2.5055299361334815\n",
      "./graph_5_16/2019-05-16 15:27:01.353104819~2019-05-16 15:42:08.532100479.txt    4.027563907907445  count: 4504  percentage: 0.09996448863636363  node count: 207  edge count: 202\n",
      "index_count: 61\n",
      "thr: 2.2343357554935936\n",
      "./graph_5_16/2019-05-16 15:42:08.532100479~2019-05-16 15:57:14.083226569.txt    4.181735096588558  count: 1983  percentage: 0.06677667025862069  node count: 118  edge count: 112\n",
      "index_count: 62\n",
      "thr: 2.713454639110089\n",
      "./graph_5_16/2019-05-16 15:57:14.083226569~2019-05-16 16:12:28.333264297.txt    4.712580602558261  count: 6444  percentage: 0.08740234375  node count: 210  edge count: 202\n",
      "index_count: 63\n",
      "thr: 2.091426189025028\n",
      "./graph_5_16/2019-05-16 16:12:28.333264297~2019-05-16 16:27:29.952915678.txt    3.6839210009062855  count: 5541  percentage: 0.0845489501953125  node count: 235  edge count: 225\n",
      "index_count: 64\n",
      "thr: 2.6672366104611243\n",
      "./graph_5_16/2019-05-16 16:27:29.952915678~2019-05-16 16:42:33.013069526.txt    4.916991556794179  count: 3522  percentage: 0.07643229166666667  node count: 73  edge count: 65\n",
      "index_count: 65\n",
      "thr: 2.328370849119478\n",
      "./graph_5_16/2019-05-16 16:42:33.013069526~2019-05-16 16:57:34.092939750.txt    4.295639734796208  count: 8832  percentage: 0.0756578947368421  node count: 376  edge count: 372\n",
      "index_count: 66\n",
      "thr: 1.828369251617836\n",
      "./graph_5_16/2019-05-16 16:57:34.092939750~2019-05-16 17:13:06.280592297.txt    3.5886359051910515  count: 3623  percentage: 0.06432883522727273  node count: 159  edge count: 151\n",
      "index_count: 67\n",
      "thr: 1.591244697458424\n",
      "./graph_5_16/2019-05-16 17:13:06.280592297~2019-05-16 17:28:10.223159643.txt    3.240888231076838  count: 6189  percentage: 0.06104995265151515  node count: 243  edge count: 255\n",
      "index_count: 68\n",
      "thr: 2.4339521502893318\n",
      "./graph_5_16/2019-05-16 17:28:10.223159643~2019-05-16 17:43:27.572851052.txt    4.495718760000627  count: 6221  percentage: 0.07408774771341463  node count: 208  edge count: 206\n",
      "index_count: 69\n",
      "thr: 2.5395472782361077\n",
      "./graph_5_16/2019-05-16 17:43:27.572851052~2019-05-16 17:58:44.112324274.txt    4.238125914187313  count: 9992  percentage: 0.09293154761904762  node count: 216  edge count: 213\n",
      "index_count: 70\n",
      "thr: 2.1613164232937687\n",
      "./graph_5_16/2019-05-16 17:58:44.112324274~2019-05-16 18:14:14.082900415.txt    3.598015963877731  count: 4660  percentage: 0.091015625  node count: 157  edge count: 151\n",
      "index_count: 71\n",
      "thr: 1.8058795398281091\n",
      "./graph_5_16/2019-05-16 18:14:14.082900415~2019-05-16 18:30:26.073055443.txt    3.684252078454021  count: 5693  percentage: 0.059144365026595744  node count: 194  edge count: 189\n",
      "index_count: 72\n",
      "thr: 2.687231120553291\n",
      "./graph_5_16/2019-05-16 18:30:26.073055443~2019-05-16 18:45:49.432070287.txt    4.5264682406148795  count: 9658  percentage: 0.09156932645631068  node count: 153  edge count: 146\n",
      "index_count: 73\n",
      "thr: 2.098171726812814\n",
      "./graph_5_16/2019-05-16 18:45:49.432070287~2019-05-16 19:01:12.193027191.txt    4.102039586289138  count: 4340  percentage: 0.06727430555555555  node count: 196  edge count: 192\n",
      "index_count: 74\n",
      "thr: 2.330333858666204\n",
      "./graph_5_16/2019-05-16 19:01:12.193027191~2019-05-16 19:16:22.673088266.txt    4.6509661188151545  count: 4412  percentage: 0.062443387681159424  node count: 129  edge count: 125\n",
      "index_count: 75\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "thr: 2.19545227887718\n",
      "./graph_5_16/2019-05-16 19:16:22.673088266~2019-05-16 19:31:32.052688344.txt    3.7039780440214445  count: 5834  percentage: 0.08503381529850747  node count: 276  edge count: 273\n",
      "index_count: 76\n",
      "thr: 2.1282683455443965\n",
      "./graph_5_16/2019-05-16 19:31:32.052688344~2019-05-16 19:46:40.442657625.txt    4.1483820859398115  count: 2964  percentage: 0.06578480113636363  node count: 86  edge count: 77\n",
      "index_count: 77\n",
      "thr: 1.7209776745274037\n",
      "./graph_5_16/2019-05-16 19:46:40.442657625~2019-05-16 20:02:06.820142338.txt    3.0599066150336665  count: 3378  percentage: 0.07171365489130435  node count: 163  edge count: 168\n",
      "index_count: 78\n",
      "thr: 2.3245108315576517\n",
      "./graph_5_16/2019-05-16 20:02:06.820142338~2019-05-16 20:17:11.412990296.txt    3.8753010489826125  count: 8057  percentage: 0.09366861979166667  node count: 310  edge count: 302\n",
      "index_count: 79\n",
      "thr: 1.8081077672077597\n",
      "./graph_5_16/2019-05-16 20:17:11.412990296~2019-05-16 20:32:27.570220441.txt    3.969411948643162  count: 2832  percentage: 0.05028409090909091  node count: 108  edge count: 103\n",
      "index_count: 80\n",
      "thr: 2.606373355980588\n",
      "node: {'subject': 'nginx'}  IDF: 5.752572638825633\n",
      "history queue: ./graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt\n",
      "./graph_5_16/2019-05-16 20:32:27.570220441~2019-05-16 20:48:38.072848659.txt    3.934422186044603  count: 7784  percentage: 0.10706426056338028  node count: 172  edge count: 164\n",
      "index_count: 81\n",
      "thr: 1.7495021724594704\n",
      "./graph_5_16/2019-05-16 20:48:38.072848659~2019-05-16 21:03:58.072828936.txt    3.1041980103301485  count: 4448  percentage: 0.07239583333333334  node count: 157  edge count: 148\n",
      "index_count: 82\n",
      "thr: 2.638029361064426\n",
      "./graph_5_16/2019-05-16 21:03:58.072828936~2019-05-16 21:19:00.930018779.txt    4.680951033495144  count: 4175  percentage: 0.07692732900943396  node count: 219  edge count: 210\n",
      "index_count: 83\n",
      "thr: 2.3910154878246384\n",
      "node: {'subject': 'php-fpm'}  IDF: 5.752572638825633\n",
      "node: {'subject': 'nginx'}  IDF: 5.752572638825633\n",
      "history queue: ./graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt\n",
      "./graph_5_16/2019-05-16 21:19:00.930018779~2019-05-16 21:34:46.231624861.txt    4.5367507085935905  count: 7799  percentage: 0.07052047164351852  node count: 340  edge count: 349\n",
      "index_count: 84\n",
      "thr: 2.1404033576713797\n",
      "node: {'subject': 'nginx'}  IDF: 5.752572638825633\n",
      "history queue: ./graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt\n",
      "./graph_5_16/2019-05-16 21:34:46.231624861~2019-05-16 21:49:46.992678639.txt    4.00370548339006  count: 6129  percentage: 0.07041590073529412  node count: 190  edge count: 182\n",
      "index_count: 85\n",
      "thr: 2.219078368216277\n",
      "node: {'subject': 'nginx'}  IDF: 5.752572638825633\n",
      "history queue: ./graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt\n",
      "./graph_5_16/2019-05-16 21:49:46.992678639~2019-05-16 22:06:14.950154813.txt    4.1270341524483225  count: 8212  percentage: 0.07862285539215687  node count: 244  edge count: 243\n",
      "index_count: 86\n",
      "thr: 2.3431967870845747\n",
      "node: {'subject': 'php-fpm'}  IDF: 5.752572638825633\n",
      "node: {'subject': 'nginx'}  IDF: 5.752572638825633\n",
      "history queue: ./graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt\n",
      "./graph_5_16/2019-05-16 22:06:14.950154813~2019-05-16 22:21:40.662702391.txt    4.707220671821367  count: 3152  percentage: 0.0615625  node count: 176  edge count: 177\n",
      "index_count: 87\n",
      "thr: 3.5576099572553375\n",
      "node: {'subject': 'nginx'}  IDF: 5.752572638825633\n",
      "node: {'subject': 'php-fpm'}  IDF: 5.752572638825633\n",
      "history queue: ./graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt\n",
      "./graph_5_16/2019-05-16 22:21:40.662702391~2019-05-16 22:36:45.602858389.txt    5.50422312040045  count: 12562  percentage: 0.10761033442982457  node count: 339  edge count: 351\n",
      "index_count: 88\n",
      "thr: 1.7284621281407575\n",
      "node: {'subject': 'php-fpm'}  IDF: 5.752572638825633\n",
      "node: {'subject': 'nginx'}  IDF: 5.752572638825633\n",
      "history queue: ./graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt\n",
      "./graph_5_16/2019-05-16 22:36:45.602858389~2019-05-16 22:51:51.220035024.txt    3.564229757328381  count: 3732  percentage: 0.057849702380952384  node count: 158  edge count: 154\n",
      "index_count: 89\n",
      "thr: 2.3601120131031887\n",
      "node: {'subject': 'php-fpm'}  IDF: 5.752572638825633\n",
      "node: {'subject': 'nginx'}  IDF: 5.752572638825633\n",
      "history queue: ./graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt\n",
      "./graph_5_16/2019-05-16 22:51:51.220035024~2019-05-16 23:07:16.890296254.txt    4.079073498252615  count: 8703  percentage: 0.0825147906553398  node count: 290  edge count: 266\n",
      "index_count: 90\n",
      "thr: 2.581423378137388\n",
      "node: {'subject': 'php-fpm'}  IDF: 5.752572638825633\n",
      "node: {'subject': 'nginx'}  IDF: 5.752572638825633\n",
      "history queue: ./graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt\n",
      "./graph_5_16/2019-05-16 23:07:16.890296254~2019-05-16 23:22:54.052353000.txt    4.581693324033824  count: 9774  percentage: 0.08522251674107142  node count: 280  edge count: 278\n",
      "index_count: 91\n",
      "thr: 2.462341809572649\n",
      "./graph_5_16/2019-05-16 23:22:54.052353000~2019-05-16 23:38:22.520220953.txt    5.023180929633095  count: 3236  percentage: 0.05356197033898305  node count: 108  edge count: 103\n",
      "index_count: 92\n",
      "thr: 1.9755881317614874\n",
      "./graph_5_16/2019-05-16 23:38:22.520220953~2019-05-16 23:53:23.392403039.txt    3.8144053572024674  count: 4992  percentage: 0.06678082191780822  node count: 186  edge count: 180\n"
     ]
    }
   ],
   "source": [
    "# node_IDF=torch.load(\"node_IDF_5_16\")\n",
    "# node_IDF=torch.load(\"node_IDF_5_9-12\")\n",
    "y_data_5_16=[]\n",
    "df_list_5_16=[]\n",
    "# node_set_list=[]\n",
    "history_list_5_16=[]\n",
    "tw_que=[]\n",
    "his_tw={}\n",
    "current_tw={}\n",
    "\n",
    "file_path_list=[]\n",
    "\n",
    "file_path=\"./graph_5_16/\"\n",
    "file_l=os.listdir(\"./graph_5_16/\")\n",
    "for i in file_l:\n",
    "    file_path_list.append(file_path+i)\n",
    "\n",
    "index_count=0\n",
    "for f_path in sorted(file_path_list):\n",
    "    f=open(f_path)\n",
    "    edge_loss_list=[]\n",
    "    edge_list=[]\n",
    "    print('index_count:',index_count)\n",
    "    \n",
    "    # Figure out which nodes are anomalous in this time window\n",
    "    for line in f:\n",
    "        l=line.strip()\n",
    "        jdata=eval(l)\n",
    "        edge_loss_list.append(jdata['loss'])\n",
    "        edge_list.append([str(jdata['srcmsg']),str(jdata['dstmsg'])])\n",
    "    df_list_5_16.append(pd.DataFrame(edge_loss_list))\n",
    "    count,loss_avg,node_set,edge_set=cal_anomaly_loss(edge_loss_list,edge_list,\"./graph_5_16/\")\n",
    "\n",
    "    current_tw['name']=f_path\n",
    "    current_tw['loss']=loss_avg\n",
    "    current_tw['index']=index_count\n",
    "    current_tw['nodeset']=node_set\n",
    "\n",
    "    # Incrementally construct the queues\n",
    "    added_que_flag=False\n",
    "    for hq in history_list_5_16:\n",
    "        for his_tw in hq:\n",
    "            if cal_set_rel_bak(current_tw['nodeset'],his_tw['nodeset'],file_list)!=0 and current_tw['name']!=his_tw['name']:\n",
    "                print(\"history queue:\",his_tw['name'])\n",
    "                # check if there are intersection between two time windows.\n",
    "                hq.append(copy.deepcopy(current_tw))\n",
    "                added_que_flag=True\n",
    "                break\n",
    "            if added_que_flag:\n",
    "                break\n",
    "    if added_que_flag is False:\n",
    "        temp_hq=[copy.deepcopy(current_tw)]\n",
    "        history_list_5_16.append(temp_hq)\n",
    "    index_count+=1\n",
    "    print( f_path,\"  \",loss_avg,\" count:\",count,\" percentage:\",count/len(edge_list),\" node count:\",len(node_set),\" edge count:\",len(edge_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt\n",
      "./graph_5_16/2019-05-16 09:36:08.903494477~2019-05-16 09:51:22.110949680.txt\n",
      "./graph_5_16/2019-05-16 09:51:22.110949680~2019-05-16 10:06:29.403713371.txt\n",
      "./graph_5_16/2019-05-16 10:06:29.403713371~2019-05-16 10:21:47.983513184.txt\n",
      "./graph_5_16/2019-05-16 20:32:27.570220441~2019-05-16 20:48:38.072848659.txt\n",
      "./graph_5_16/2019-05-16 21:19:00.930018779~2019-05-16 21:34:46.231624861.txt\n",
      "./graph_5_16/2019-05-16 21:34:46.231624861~2019-05-16 21:49:46.992678639.txt\n",
      "./graph_5_16/2019-05-16 21:49:46.992678639~2019-05-16 22:06:14.950154813.txt\n",
      "./graph_5_16/2019-05-16 22:06:14.950154813~2019-05-16 22:21:40.662702391.txt\n",
      "./graph_5_16/2019-05-16 22:21:40.662702391~2019-05-16 22:36:45.602858389.txt\n",
      "./graph_5_16/2019-05-16 22:36:45.602858389~2019-05-16 22:51:51.220035024.txt\n",
      "./graph_5_16/2019-05-16 22:51:51.220035024~2019-05-16 23:07:16.890296254.txt\n",
      "./graph_5_16/2019-05-16 23:07:16.890296254~2019-05-16 23:22:54.052353000.txt\n",
      "2262998543.4288015\n"
     ]
    }
   ],
   "source": [
    "name_list=[]\n",
    "for hl in history_list_5_16:\n",
    "    loss_count=0\n",
    "    for hq in hl:\n",
    "        if loss_count==0:\n",
    "            loss_count=(loss_count+1)*(hq['loss']+1)\n",
    "        else:\n",
    "            loss_count=(loss_count)*(hq['loss']+1)\n",
    "#     name_list=[]\n",
    "    if loss_count>100:\n",
    "        name_list=[]\n",
    "        for i in hl:\n",
    "            name_list.append(i['name'])\n",
    "            print(i['name'])\n",
    "#         print(name_list)\n",
    "        for i in name_list:\n",
    "            pred_label[i]=1\n",
    "        print(loss_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Anomaly Detection 5-17"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "index_count: 0\n",
      "thr: 1.1795852483215774\n",
      "./graph_5_17/2019-05-17 00:00:00.011404169~2019-05-17 00:15:19.121412431.txt    3.0619232276393005  count: 3086  percentage: 0.04305245535714286  node count: 268  edge count: 243\n",
      "index_count: 1\n",
      "thr: 2.0078692964972067\n",
      "./graph_5_17/2019-05-17 00:15:19.121412431~2019-05-17 00:30:22.942273426.txt    3.4775138841842814  count: 4791  percentage: 0.08066742995689655  node count: 186  edge count: 177\n",
      "index_count: 2\n",
      "thr: 1.2057315811286506\n",
      "./graph_5_17/2019-05-17 00:30:22.942273426~2019-05-17 00:45:36.772457892.txt    2.571637964544053  count: 4197  percentage: 0.0546484375  node count: 122  edge count: 119\n",
      "index_count: 3\n",
      "thr: 2.7981392522286805\n",
      "./graph_5_17/2019-05-17 00:45:36.772457892~2019-05-17 01:00:43.031312698.txt    4.06124922693654  count: 7194  percentage: 0.12112742456896551  node count: 53  edge count: 48\n",
      "index_count: 4\n",
      "thr: 2.038159080075502\n",
      "./graph_5_17/2019-05-17 01:00:43.031312698~2019-05-17 01:15:44.002530354.txt    3.528074391248113  count: 6141  percentage: 0.08329264322916667  node count: 146  edge count: 143\n",
      "index_count: 5\n",
      "thr: 2.1171245241864307\n",
      "./graph_5_17/2019-05-17 01:15:44.002530354~2019-05-17 01:30:47.512391580.txt    3.421335553296307  count: 8249  percentage: 0.09154163707386363  node count: 332  edge count: 338\n",
      "index_count: 6\n",
      "thr: 2.219239356463066\n",
      "./graph_5_17/2019-05-17 01:30:47.512391580~2019-05-17 01:46:06.532627237.txt    4.356629769956699  count: 2344  percentage: 0.06358506944444445  node count: 68  edge count: 61\n",
      "index_count: 7\n",
      "thr: 1.9717099950858321\n",
      "./graph_5_17/2019-05-17 01:46:06.532627237~2019-05-17 02:01:06.862180143.txt    3.752994552099521  count: 4802  percentage: 0.06896254595588236  node count: 123  edge count: 119\n",
      "index_count: 8\n",
      "thr: 3.193269847614746\n",
      "./graph_5_17/2019-05-17 02:01:06.862180143~2019-05-17 02:17:12.251503432.txt    5.122195178620646  count: 4526  percentage: 0.10523623511904762  node count: 83  edge count: 75\n",
      "index_count: 9\n",
      "thr: 1.9586488151887957\n",
      "./graph_5_17/2019-05-17 02:17:12.251503432~2019-05-17 02:33:34.082266094.txt    4.281788875210263  count: 3783  percentage: 0.0568359375  node count: 222  edge count: 211\n",
      "index_count: 10\n",
      "thr: 1.418794927333824\n",
      "./graph_5_17/2019-05-17 02:33:34.082266094~2019-05-17 02:48:48.272633335.txt    2.9239908785199527  count: 5318  percentage: 0.05969378591954023  node count: 270  edge count: 292\n",
      "index_count: 11\n",
      "thr: 1.3378817060029196\n",
      "./graph_5_17/2019-05-17 02:48:48.272633335~2019-05-17 03:04:02.082047814.txt    3.288433169929493  count: 2022  percentage: 0.042012965425531915  node count: 131  edge count: 126\n",
      "index_count: 12\n",
      "thr: 1.260702629456264\n",
      "./graph_5_17/2019-05-17 03:04:02.082047814~2019-05-17 03:19:06.352094009.txt    2.9330739529769647  count: 1711  percentage: 0.042843549679487176  node count: 62  edge count: 56\n",
      "index_count: 13\n",
      "thr: 1.181551807946509\n",
      "./graph_5_17/2019-05-17 03:19:06.352094009~2019-05-17 03:35:22.072176911.txt    3.1524852080050683  count: 4615  percentage: 0.037872570903361345  node count: 210  edge count: 204\n",
      "index_count: 14\n",
      "thr: 1.7160553825809406\n",
      "./graph_5_17/2019-05-17 03:35:22.072176911~2019-05-17 03:50:24.232420260.txt    3.267942933603487  count: 6016  percentage: 0.07532051282051282  node count: 167  edge count: 156\n",
      "index_count: 15\n",
      "thr: 1.893668610963453\n",
      "./graph_5_17/2019-05-17 03:50:24.232420260~2019-05-17 04:05:38.062092796.txt    3.333504071100858  count: 4351  percentage: 0.08498046875  node count: 93  edge count: 85\n",
      "index_count: 16\n",
      "thr: 1.3018082685210122\n",
      "./graph_5_17/2019-05-17 04:05:38.062092796~2019-05-17 04:20:39.512164044.txt    3.2993619695834675  count: 1916  percentage: 0.04563643292682927  node count: 167  edge count: 157\n",
      "index_count: 17\n",
      "thr: 1.983149880077751\n",
      "./graph_5_17/2019-05-17 04:20:39.512164044~2019-05-17 04:36:14.272391092.txt    3.6734198399525475  count: 5424  percentage: 0.07676630434782608  node count: 141  edge count: 132\n",
      "index_count: 18\n",
      "thr: 2.3081552758587662\n",
      "./graph_5_17/2019-05-17 04:36:14.272391092~2019-05-17 04:51:18.351942306.txt    5.026468016359035  count: 3877  percentage: 0.06009734623015873  node count: 185  edge count: 180\n",
      "index_count: 19\n",
      "thr: 2.220438038234416\n",
      "./graph_5_17/2019-05-17 04:51:18.351942306~2019-05-17 05:06:35.492322242.txt    4.159968860695759  count: 3840  percentage: 0.07653061224489796  node count: 48  edge count: 43\n",
      "index_count: 20\n",
      "thr: 2.212455284935354\n",
      "./graph_5_17/2019-05-17 05:06:35.492322242~2019-05-17 05:21:53.681001911.txt    4.328597842316421  count: 4390  percentage: 0.06804935515873016  node count: 165  edge count: 160\n",
      "index_count: 21\n",
      "thr: 1.449812556051354\n",
      "./graph_5_17/2019-05-17 05:21:53.681001911~2019-05-17 05:37:20.351873432.txt    3.054243872574718  count: 1702  percentage: 0.055403645833333334  node count: 80  edge count: 68\n",
      "index_count: 22\n",
      "thr: 1.6585272448277106\n",
      "./graph_5_17/2019-05-17 05:37:20.351873432~2019-05-17 05:53:46.081922263.txt    3.149186379597362  count: 3712  percentage: 0.06839622641509434  node count: 89  edge count: 83\n",
      "index_count: 23\n",
      "thr: 1.8553543597362947\n",
      "./graph_5_17/2019-05-17 05:53:46.081922263~2019-05-17 06:09:26.061933299.txt    3.96846846116623  count: 957  percentage: 0.054974724264705885  node count: 100  edge count: 99\n",
      "index_count: 24\n",
      "thr: 1.591735339862725\n",
      "./graph_5_17/2019-05-17 06:09:26.061933299~2019-05-17 06:24:48.061908897.txt    3.9823055304441937  count: 771  percentage: 0.04428998161764706  node count: 98  edge count: 92\n",
      "index_count: 25\n",
      "thr: 1.4820836498493903\n",
      "./graph_5_17/2019-05-17 06:24:48.061908897~2019-05-17 06:40:14.072014912.txt    2.9341354013182395  count: 688  percentage: 0.055989583333333336  node count: 54  edge count: 46\n",
      "index_count: 26\n",
      "thr: 2.3909686365380605\n",
      "./graph_5_17/2019-05-17 06:40:14.072014912~2019-05-17 06:55:30.760988446.txt    3.8685429661086865  count: 1988  percentage: 0.09244791666666667  node count: 116  edge count: 110\n",
      "index_count: 27\n",
      "thr: 1.9329094116318224\n",
      "./graph_5_17/2019-05-17 06:55:30.760988446~2019-05-17 07:11:16.051875332.txt    4.006176845968976  count: 939  percentage: 0.050944010416666664  node count: 78  edge count: 69\n",
      "index_count: 28\n",
      "thr: 2.7234936851869342\n",
      "./graph_5_17/2019-05-17 07:11:16.051875332~2019-05-17 07:27:34.071942527.txt    4.429187195837541  count: 1391  percentage: 0.1044921875  node count: 71  edge count: 64\n",
      "index_count: 29\n",
      "thr: 1.883291629310349\n",
      "./graph_5_17/2019-05-17 07:27:34.071942527~2019-05-17 07:44:04.071921360.txt    3.9354237238843957  count: 1144  percentage: 0.06571691176470588  node count: 36  edge count: 35\n",
      "index_count: 30\n",
      "thr: 0.5611294865452087\n",
      "./graph_5_17/2019-05-17 07:44:04.071921360~2019-05-17 07:59:22.051873504.txt    2.6401137350773323  count: 98  percentage: 0.013671875  node count: 11  edge count: 10\n",
      "index_count: 31\n",
      "thr: 0.5656048073926975\n",
      "./graph_5_17/2019-05-17 07:59:22.051873504~2019-05-17 08:14:56.081692880.txt    2.154405289927855  count: 151  percentage: 0.0184326171875  node count: 23  edge count: 20\n",
      "index_count: 32\n",
      "thr: 0.5502282518382189\n",
      "./graph_5_17/2019-05-17 08:14:56.081692880~2019-05-17 08:30:22.081640414.txt    2.866240684281696  count: 88  percentage: 0.012276785714285714  node count: 7  edge count: 7\n",
      "index_count: 33\n",
      "thr: 0.5425414279063174\n",
      "./graph_5_17/2019-05-17 08:30:22.081640414~2019-05-17 08:45:38.051874367.txt    2.456933947971889  count: 105  percentage: 0.0146484375  node count: 11  edge count: 9\n",
      "index_count: 34\n",
      "thr: 1.2307582897188412\n",
      "./graph_5_17/2019-05-17 08:45:38.051874367~2019-05-17 09:01:32.730750122.txt    4.401195467873053  count: 176  percentage: 0.021484375  node count: 60  edge count: 58\n",
      "index_count: 35\n",
      "thr: 1.521584666107688\n",
      "./graph_5_17/2019-05-17 09:01:32.730750122~2019-05-17 09:16:44.320659825.txt    3.1568238742467836  count: 2132  percentage: 0.056271114864864864  node count: 135  edge count: 125\n",
      "index_count: 36\n",
      "thr: 2.3552468782918767\n",
      "./graph_5_17/2019-05-17 09:16:44.320659825~2019-05-17 09:31:47.131585900.txt    4.031215931568594  count: 5917  percentage: 0.08138479313380281  node count: 116  edge count: 111\n",
      "index_count: 37\n",
      "thr: 1.9926714127902119\n",
      "./graph_5_17/2019-05-17 09:31:47.131585900~2019-05-17 09:46:50.071747334.txt    3.441770499758893  count: 4748  percentage: 0.08586516203703703  node count: 101  edge count: 89\n",
      "index_count: 38\n",
      "thr: 1.8639097705147067\n",
      "./graph_5_17/2019-05-17 09:46:50.071747334~2019-05-17 10:02:11.321524261.txt    3.843318432123552  count: 4678  percentage: 0.05782733386075949  node count: 221  edge count: 222\n",
      "index_count: 39\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "thr: 2.0365885415118274\n",
      "./graph_5_17/2019-05-17 10:02:11.321524261~2019-05-17 10:17:26.881636687.txt    3.7463298083573537  count: 6049  percentage: 0.07876302083333334  node count: 152  edge count: 148\n",
      "index_count: 40\n",
      "thr: 2.5159587011102156\n",
      "node: {'subject': 'nginx'}  IDF: 5.752572638825633\n",
      "history queue: ./graph_5_17/2019-05-17 10:02:11.321524261~2019-05-17 10:17:26.881636687.txt\n",
      "./graph_5_17/2019-05-17 10:17:26.881636687~2019-05-17 10:32:38.131495470.txt    4.285088405297908  count: 6435  percentage: 0.09107506793478261  node count: 246  edge count: 243\n",
      "index_count: 41\n",
      "thr: 1.5953392271406703\n",
      "node: {'subject': 'nginx'}  IDF: 5.752572638825633\n",
      "history queue: ./graph_5_17/2019-05-17 10:02:11.321524261~2019-05-17 10:17:26.881636687.txt\n",
      "./graph_5_17/2019-05-17 10:32:38.131495470~2019-05-17 10:48:02.091564015.txt    3.1024392537701617  count: 1783  percentage: 0.06448929398148148  node count: 129  edge count: 128\n",
      "index_count: 42\n",
      "thr: 2.0637295854056323\n",
      "./graph_5_17/2019-05-17 10:48:02.091564015~2019-05-17 11:04:49.619210641.txt    3.6359113498018956  count: 6126  percentage: 0.08308919270833333  node count: 276  edge count: 273\n",
      "index_count: 43\n",
      "thr: 2.39763985343604\n",
      "./graph_5_17/2019-05-17 11:04:49.619210641~2019-05-17 11:19:49.740854239.txt    4.211443152787586  count: 7459  percentage: 0.08372620330459771  node count: 178  edge count: 169\n",
      "index_count: 44\n",
      "thr: 2.5515139428536444\n",
      "./graph_5_17/2019-05-17 11:19:49.740854239~2019-05-17 11:35:22.890857355.txt    4.231330452053228  count: 5425  percentage: 0.09134226831896551  node count: 209  edge count: 206\n",
      "index_count: 45\n",
      "thr: 2.188633276173352\n",
      "./graph_5_17/2019-05-17 11:35:22.890857355~2019-05-17 11:50:25.769134767.txt    3.83643236539951  count: 5413  percentage: 0.08009292140151515  node count: 174  edge count: 170\n",
      "index_count: 46\n",
      "thr: 1.7181023187396645\n",
      "./graph_5_17/2019-05-17 11:50:25.769134767~2019-05-17 12:05:34.151631198.txt    3.062876215546164  count: 8278  percentage: 0.08334004510309279  node count: 311  edge count: 302\n",
      "index_count: 47\n",
      "thr: 2.4598344842152073\n",
      "./graph_5_17/2019-05-17 12:05:34.151631198~2019-05-17 12:20:48.219175447.txt    4.280897234879631  count: 3706  percentage: 0.08416606104651163  node count: 154  edge count: 150\n",
      "index_count: 48\n",
      "thr: 1.9046369472797011\n",
      "./graph_5_17/2019-05-17 12:20:48.219175447~2019-05-17 12:36:20.051747107.txt    3.520807017433309  count: 5350  percentage: 0.07156999143835617  node count: 172  edge count: 168\n",
      "index_count: 49\n",
      "thr: 1.6337122144625424\n",
      "./graph_5_17/2019-05-17 12:36:20.051747107~2019-05-17 12:51:28.051740428.txt    3.1170614948114483  count: 3074  percentage: 0.0714750744047619  node count: 192  edge count: 185\n",
      "index_count: 50\n",
      "thr: 1.619474454096598\n",
      "./graph_5_17/2019-05-17 12:51:28.051740428~2019-05-17 13:06:40.331492736.txt    2.949101049189682  count: 4541  percentage: 0.07918875558035714  node count: 86  edge count: 78\n",
      "index_count: 51\n",
      "thr: 2.2619026815108456\n",
      "./graph_5_17/2019-05-17 13:06:40.331492736~2019-05-17 13:21:54.750460761.txt    3.9437422123099974  count: 9362  percentage: 0.08625073702830188  node count: 358  edge count: 356\n",
      "index_count: 52\n",
      "thr: 1.8506438456947636\n",
      "./graph_5_17/2019-05-17 13:21:54.750460761~2019-05-17 13:37:08.741478317.txt    3.499115836418281  count: 3540  percentage: 0.06778492647058823  node count: 217  edge count: 206\n",
      "index_count: 53\n",
      "thr: 1.6783609356992324\n",
      "./graph_5_17/2019-05-17 13:37:08.741478317~2019-05-17 13:52:20.000650535.txt    3.684765367233774  count: 5183  percentage: 0.050615234375  node count: 204  edge count: 200\n",
      "index_count: 54\n",
      "thr: 1.6239860594302664\n",
      "./graph_5_17/2019-05-17 13:52:20.000650535~2019-05-17 14:07:34.051622439.txt    2.794246771949494  count: 3340  percentage: 0.0836338141025641  node count: 95  edge count: 92\n",
      "index_count: 55\n",
      "thr: 2.223293981306476\n",
      "./graph_5_17/2019-05-17 14:07:34.051622439~2019-05-17 14:22:39.531584558.txt    4.1229363120788705  count: 4049  percentage: 0.06701867055084745  node count: 390  edge count: 387\n",
      "index_count: 56\n",
      "thr: 2.831233807621963\n",
      "./graph_5_17/2019-05-17 14:22:39.531584558~2019-05-17 14:37:39.821475906.txt    4.579593731497319  count: 9540  percentage: 0.09224164603960396  node count: 254  edge count: 248\n",
      "index_count: 57\n",
      "thr: 1.7198249153434269\n",
      "./graph_5_17/2019-05-17 14:37:39.821475906~2019-05-17 14:52:40.948619952.txt    3.302160103226253  count: 4199  percentage: 0.06722272028688525  node count: 202  edge count: 195\n",
      "index_count: 58\n",
      "thr: 2.2893084265459827\n",
      "./graph_5_17/2019-05-17 14:52:40.948619952~2019-05-17 15:07:56.121477590.txt    3.944138239590896  count: 9355  percentage: 0.08700706845238096  node count: 277  edge count: 270\n",
      "index_count: 59\n",
      "thr: 2.3607382988854577\n",
      "./graph_5_17/2019-05-17 15:07:56.121477590~2019-05-17 15:23:16.091314858.txt    4.004666056411377  count: 11339  percentage: 0.09002635924796748  node count: 268  edge count: 267\n",
      "index_count: 60\n",
      "thr: 1.7234099605872493\n",
      "./graph_5_17/2019-05-17 15:23:16.091314858~2019-05-17 15:38:50.341411090.txt    3.5292086949457926  count: 2352  percentage: 0.06044407894736842  node count: 178  edge count: 168\n",
      "index_count: 61\n",
      "thr: 1.787970980557266\n",
      "./graph_5_17/2019-05-17 15:38:50.341411090~2019-05-17 15:54:28.051468310.txt    3.1436937630129687  count: 5088  percentage: 0.0828125  node count: 171  edge count: 168\n",
      "index_count: 62\n",
      "thr: 1.9697081959795746\n",
      "node: {'file': '/var/spool/clientmqueue/./dfx4HK404U035644'}  IDF: 5.752572638825633\n",
      "history queue: ./graph_5_17/2019-05-17 09:31:47.131585900~2019-05-17 09:46:50.071747334.txt\n",
      "./graph_5_17/2019-05-17 15:54:28.051468310~2019-05-17 16:09:30.211020620.txt    3.811279620273887  count: 4127  percentage: 0.07070655153508772  node count: 187  edge count: 176\n",
      "index_count: 63\n",
      "thr: 2.3159319760518717\n",
      "./graph_5_17/2019-05-17 16:09:30.211020620~2019-05-17 16:24:30.611175860.txt    4.2871219866743715  count: 5136  percentage: 0.07486007462686567  node count: 287  edge count: 278\n",
      "index_count: 64\n",
      "thr: 1.270885103196039\n",
      "./graph_5_17/2019-05-17 16:24:30.611175860~2019-05-17 16:39:33.741234335.txt    2.8839904580159645  count: 3738  percentage: 0.050699869791666664  node count: 202  edge count: 197\n",
      "index_count: 65\n",
      "thr: 1.4501950250901794\n",
      "./graph_5_17/2019-05-17 16:39:33.741234335~2019-05-17 16:54:49.968820600.txt    3.1493521014386525  count: 1803  percentage: 0.048909505208333336  node count: 213  edge count: 209\n",
      "index_count: 66\n",
      "thr: 2.01365946570682\n",
      "./graph_5_17/2019-05-17 16:54:49.968820600~2019-05-17 17:09:59.490279010.txt    3.7864713781834656  count: 8059  percentage: 0.07026890345982142  node count: 330  edge count: 340\n",
      "index_count: 67\n",
      "thr: 2.4242663264363324\n",
      "./graph_5_17/2019-05-17 17:09:59.490279010~2019-05-17 17:26:04.070998589.txt    3.988821056600167  count: 4691  percentage: 0.09543863932291667  node count: 152  edge count: 147\n",
      "index_count: 68\n",
      "thr: 0.4830412666103598\n",
      "./graph_5_17/2019-05-17 17:26:04.070998589~2019-05-17 17:42:18.071078452.txt    3.148775256261593  count: 41  percentage: 0.0080078125  node count: 5  edge count: 5\n"
     ]
    }
   ],
   "source": [
    "# node_IDF=torch.load(\"node_IDF_5_17\")\n",
    "# node_IDF=torch.load(\"node_IDF_5_9-12\")\n",
    "y_data_5_17=[]\n",
    "df_list_5_17=[]\n",
    "# node_set_list=[]\n",
    "history_list_5_17=[]\n",
    "tw_que=[]\n",
    "his_tw={}\n",
    "current_tw={}\n",
    "\n",
    "loss_list_5_17=[]\n",
    "\n",
    "file_path_list=[]\n",
    "\n",
    "file_path=\"./graph_5_17/\"\n",
    "file_l=os.listdir(\"./graph_5_17/\")\n",
    "for i in file_l:\n",
    "    file_path_list.append(file_path+i)\n",
    "\n",
    "index_count=0\n",
    "for f_path in sorted(file_path_list):\n",
    "    f=open(f_path)\n",
    "    edge_loss_list=[]\n",
    "    edge_list=[]\n",
    "    print('index_count:',index_count)\n",
    "    \n",
    "    for line in f:\n",
    "        l=line.strip()\n",
    "        jdata=eval(l)\n",
    "        edge_loss_list.append(jdata['loss'])\n",
    "        edge_list.append([str(jdata['srcmsg']),str(jdata['dstmsg'])])\n",
    "    df_list_5_17.append(pd.DataFrame(edge_loss_list))\n",
    "    count,loss_avg,node_set,edge_set=cal_anomaly_loss(edge_loss_list,edge_list,\"./clear_data/graph_5_17/\")\n",
    "\n",
    "    current_tw['name']=f_path\n",
    "    current_tw['loss']=loss_avg\n",
    "    current_tw['index']=index_count\n",
    "    current_tw['nodeset']=node_set\n",
    "\n",
    "    added_que_flag=False\n",
    "    for hq in history_list_5_17:\n",
    "        for his_tw in hq:\n",
    "            if cal_set_rel_bak(current_tw['nodeset'],his_tw['nodeset'],file_list)!=0 and current_tw['name']!=his_tw['name']:\n",
    "                print(\"history queue:\",his_tw['name'])\n",
    "\n",
    "                hq.append(copy.deepcopy(current_tw))\n",
    "                added_que_flag=True\n",
    "                break\n",
    "            if added_que_flag:\n",
    "                break\n",
    "    if added_que_flag is False:\n",
    "        temp_hq=[copy.deepcopy(current_tw)]\n",
    "        history_list_5_17.append(temp_hq)\n",
    "    index_count+=1\n",
    "    loss_list_5_17.append(loss_avg)\n",
    "    print( f_path,\"  \",loss_avg,\" count:\",count,\" percentage:\",count/len(edge_list),\" node count:\",len(node_set),\" edge count:\",len(edge_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./graph_5_17/2019-05-17 10:02:11.321524261~2019-05-17 10:17:26.881636687.txt\n",
      "./graph_5_17/2019-05-17 10:17:26.881636687~2019-05-17 10:32:38.131495470.txt\n",
      "./graph_5_17/2019-05-17 10:32:38.131495470~2019-05-17 10:48:02.091564015.txt\n",
      "102.90875594149468\n"
     ]
    }
   ],
   "source": [
    "name_list=[]\n",
    "for hl in history_list_5_17:\n",
    "    loss_count=0\n",
    "    for hq in hl:\n",
    "        if loss_count==0:\n",
    "            loss_count=(loss_count+1)*(hq['loss']+1)\n",
    "        else:\n",
    "            loss_count=(loss_count)*(hq['loss']+1)\n",
    "#     name_list=[]\n",
    "    if loss_count>100:\n",
    "        name_list=[]\n",
    "        for i in hl:\n",
    "            name_list.append(i['name']) \n",
    "            print(i['name'])\n",
    "        for i in name_list:\n",
    "            pred_label[i]=1\n",
    "        print(loss_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import average_precision_score, roc_auc_score\n",
    "\n",
    "# from sklearn.metrics import plot_roc_curve,roc_curve,auc,roc_auc_score\n",
    "import torch\n",
    "from sklearn import preprocessing\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "def plot_thr():\n",
    "    np.seterr(invalid='ignore')\n",
    "    step=0.01\n",
    "    thr_list=torch.arange(-5,5,step)\n",
    "    \n",
    "    \n",
    "\n",
    "    precision_list=[]\n",
    "    recall_list=[]\n",
    "    fscore_list=[]\n",
    "    accuracy_list=[]\n",
    "    auc_val_list=[]\n",
    "    for thr in thr_list:\n",
    "        threshold=thr\n",
    "        y_prediction=[]\n",
    "        for i in y_test_scores:\n",
    "            if i >threshold:\n",
    "                y_prediction.append(1)\n",
    "            else:\n",
    "                y_prediction.append(0)\n",
    "        precision,recall,fscore,accuracy,auc_val=classifier_evaluation(y_test, y_prediction)   \n",
    "        precision_list.append(float(precision))\n",
    "        recall_list.append(float(recall))\n",
    "        fscore_list.append(float(fscore))\n",
    "        accuracy_list.append(float(accuracy))\n",
    "        auc_val_list.append(float(auc_val))\n",
    "\n",
    "    max_fscore=max(fscore_list)\n",
    "    max_fscore_index=fscore_list.index(max_fscore)\n",
    "    print(max_fscore_index)\n",
    "    print(\"max threshold:\",thr_list[max_fscore_index])\n",
    "    print('precision:',precision_list[max_fscore_index])\n",
    "    print('recall:',recall_list[max_fscore_index])\n",
    "    print('fscore:',fscore_list[max_fscore_index])\n",
    "    print('accuracy:',accuracy_list[max_fscore_index])    \n",
    "    print('auc:',auc_val_list[max_fscore_index])\n",
    "\n",
    "    plt.plot(thr_list,precision_list,color='red',label='precision',linewidth=2.0,linestyle='-')\n",
    "    plt.plot(thr_list,recall_list,color='orange',label='recall',linewidth=2.0,linestyle='solid')\n",
    "    plt.plot(thr_list,fscore_list,color='y',label='F-score',linewidth=2.0,linestyle='dashed')\n",
    "    plt.plot(thr_list,accuracy_list,color='g',label='accuracy',linewidth=2.0,linestyle='dashdot')\n",
    "    plt.plot(thr_list,auc_val_list,color='b',label='auc_val',linewidth=2.0,linestyle='dotted')\n",
    "\n",
    "\n",
    "    plt.xlabel(\"Threshold\", fontdict={'size': 16})\n",
    "    plt.ylabel(\"Rate\", fontdict={'size': 16})\n",
    "    plt.title(\"Different evaluation Indicators by varying threshold value\", fontdict={'size': 12})\n",
    "    plt.legend(loc='best', fontsize=12, markerscale=0.5)\n",
    "    plt.show()\n",
    "\n",
    "def classifier_evaluation(y_test, y_test_pred):\n",
    "    # groundtruth, pred_value\n",
    "    tn, fp, fn, tp =confusion_matrix(y_test, y_test_pred).ravel()\n",
    "#     tn+=100\n",
    "#     print(clf_name,\" : \")\n",
    "    print('tn:',tn)\n",
    "    print('fp:',fp)\n",
    "    print('fn:',fn)\n",
    "    print('tp:',tp)\n",
    "    precision=tp/(tp+fp)\n",
    "    recall=tp/(tp+fn)\n",
    "    accuracy=(tp+tn)/(tp+tn+fp+fn)\n",
    "    fscore=2*(precision*recall)/(precision+recall)    \n",
    "    auc_val=roc_auc_score(y_test, y_test_pred)\n",
    "    print(\"precision:\",precision)\n",
    "    print(\"recall:\",recall)\n",
    "    print(\"fscore:\",fscore)\n",
    "    print(\"accuracy:\",accuracy)\n",
    "    print(\"auc_val:\",auc_val)\n",
    "    return precision,recall,fscore,accuracy,auc_val\n",
    "\n",
    "def minmax(data):\n",
    "    min_val=min(data)\n",
    "    max_val=max(data)\n",
    "    ans=[]\n",
    "    for i in data:\n",
    "        ans.append((i-min_val)/(max_val-min_val))\n",
    "    return ans\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# label generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels={}\n",
    "\n",
    "filelist = os.listdir(\"graph_5_15\")\n",
    "for f in filelist:\n",
    "    labels[\"./graph_5_15/\"+f]=0\n",
    "\n",
    "    \n",
    "filelist = os.listdir(\"graph_5_16\")\n",
    "for f in filelist:\n",
    "    labels[\"./graph_5_16/\"+f]=0\n",
    "\n",
    "    \n",
    "filelist = os.listdir(\"graph_5_17\")\n",
    "for f in filelist:\n",
    "    labels[\"./graph_5_17/\"+f]=0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "attack_list=[\n",
    "    './graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt', \n",
    " './graph_5_16/2019-05-16 09:36:08.903494477~2019-05-16 09:51:22.110949680.txt', \n",
    " './graph_5_16/2019-05-16 09:51:22.110949680~2019-05-16 10:06:29.403713371.txt', \n",
    " './graph_5_16/2019-05-16 10:06:29.403713371~2019-05-16 10:21:47.983513184.txt', \n",
    "\n",
    "# Here are the \"fake\" FP time windows described in Section 5.2 in the paper.\n",
    " './graph_5_16/2019-05-16 20:32:27.570220441~2019-05-16 20:48:38.072848659.txt', \n",
    " './graph_5_16/2019-05-16 21:19:00.930018779~2019-05-16 21:34:46.231624861.txt', \n",
    " './graph_5_16/2019-05-16 21:34:46.231624861~2019-05-16 21:49:46.992678639.txt', \n",
    " './graph_5_16/2019-05-16 21:49:46.992678639~2019-05-16 22:06:14.950154813.txt', \n",
    " './graph_5_16/2019-05-16 22:06:14.950154813~2019-05-16 22:21:40.662702391.txt', \n",
    " './graph_5_16/2019-05-16 22:21:40.662702391~2019-05-16 22:36:45.602858389.txt', \n",
    " './graph_5_16/2019-05-16 22:36:45.602858389~2019-05-16 22:51:51.220035024.txt', \n",
    " './graph_5_16/2019-05-16 22:51:51.220035024~2019-05-16 23:07:16.890296254.txt', \n",
    " './graph_5_16/2019-05-16 23:07:16.890296254~2019-05-16 23:22:54.052353000.txt',\n",
    "\n",
    "    \n",
    "    './graph_5_17/2019-05-17 10:02:11.321524261~2019-05-17 10:17:26.881636687.txt', \n",
    " './graph_5_17/2019-05-17 10:17:26.881636687~2019-05-17 10:32:38.131495470.txt', \n",
    " './graph_5_17/2019-05-17 10:32:38.131495470~2019-05-17 10:48:02.091564015.txt'\n",
    "]\n",
    "\n",
    "\n",
    "# 结合 Rao的文档，对GT文档进行补充。检测出的准确率非常高。\n",
    "\n",
    "for i in attack_list:\n",
    "    labels[i]=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'./graph_5_15/2019-05-15 10:03:41.465554650~2019-05-15 10:18:48.315349922.txt': 0,\n",
       " './graph_5_15/2019-05-15 15:11:29.355153391~2019-05-15 15:26:30.315069987.txt': 0,\n",
       " './graph_5_15/2019-05-15 14:40:58.314993755~2019-05-15 14:56:22.325270164.txt': 0,\n",
       " './graph_5_15/2019-05-15 14:56:22.325270164~2019-05-15 15:11:29.355153391.txt': 0,\n",
       " './graph_5_15/2019-05-15 23:04:18.584315206~2019-05-15 23:19:21.464426330.txt': 0,\n",
       " './graph_5_15/2019-05-15 21:03:34.932076311~2019-05-15 21:18:36.594847325.txt': 0,\n",
       " './graph_5_15/2019-05-15 00:15:13.743767966~2019-05-15 00:30:15.346246304.txt': 0,\n",
       " './graph_5_15/2019-05-15 00:45:38.935375235~2019-05-15 01:00:49.986348448.txt': 0,\n",
       " './graph_5_15/2019-05-15 18:30:12.692563554~2019-05-15 18:45:36.312255908.txt': 0,\n",
       " './graph_5_15/2019-05-15 07:39:16.336018368~2019-05-15 07:55:36.325862117.txt': 0,\n",
       " './graph_5_15/2019-05-15 10:34:18.902982618~2019-05-15 10:49:19.662777306.txt': 0,\n",
       " './graph_5_15/2019-05-15 08:46:56.325786951~2019-05-15 09:02:05.234725465.txt': 0,\n",
       " './graph_5_15/2019-05-15 12:53:47.114274288~2019-05-15 13:08:48.445537909.txt': 0,\n",
       " './graph_5_15/2019-05-15 15:26:30.315069987~2019-05-15 15:41:58.325198150.txt': 0,\n",
       " './graph_5_15/2019-05-15 17:44:50.054854387~2019-05-15 17:59:50.764934674.txt': 0,\n",
       " './graph_5_15/2019-05-15 22:49:14.843539280~2019-05-15 23:04:18.584315206.txt': 0,\n",
       " './graph_5_15/2019-05-15 01:46:35.756168353~2019-05-15 02:02:03.445171413.txt': 0,\n",
       " './graph_5_15/2019-05-15 10:18:48.315349922~2019-05-15 10:34:18.902982618.txt': 0,\n",
       " './graph_5_15/2019-05-15 16:12:04.095060867~2019-05-15 16:27:36.075256581.txt': 0,\n",
       " './graph_5_15/2019-05-15 06:00:28.326014326~2019-05-15 06:17:32.325991084.txt': 0,\n",
       " './graph_5_15/2019-05-15 06:33:44.315665234~2019-05-15 06:49:36.336078067.txt': 0,\n",
       " './graph_5_15/2019-05-15 22:34:04.714368495~2019-05-15 22:49:14.843539280.txt': 0,\n",
       " './graph_5_15/2019-05-15 07:55:36.325862117~2019-05-15 08:12:44.335538340.txt': 0,\n",
       " './graph_5_15/2019-05-15 20:33:20.173943586~2019-05-15 20:48:26.344519703.txt': 0,\n",
       " './graph_5_15/2019-05-15 11:04:34.935316812~2019-05-15 11:20:14.345345440.txt': 0,\n",
       " './graph_5_15/2019-05-15 13:39:04.835250858~2019-05-15 13:54:20.623015847.txt': 0,\n",
       " './graph_5_15/2019-05-15 18:45:36.312255908~2019-05-15 19:00:43.932284310.txt': 0,\n",
       " './graph_5_15/2019-05-15 07:05:22.335601187~2019-05-15 07:23:00.336037899.txt': 0,\n",
       " './graph_5_15/2019-05-15 11:51:01.435516598~2019-05-15 12:07:06.345256228.txt': 0,\n",
       " './graph_5_15/2019-05-15 05:44:40.326005052~2019-05-15 06:00:28.326014326.txt': 0,\n",
       " './graph_5_15/2019-05-15 03:04:19.506191624~2019-05-15 03:19:20.336382785.txt': 0,\n",
       " './graph_5_15/2019-05-15 21:33:37.744873139~2019-05-15 21:48:38.194423026.txt': 0,\n",
       " './graph_5_15/2019-05-15 01:31:15.876102508~2019-05-15 01:46:35.756168353.txt': 0,\n",
       " './graph_5_15/2019-05-15 14:25:00.732554067~2019-05-15 14:40:58.314993755.txt': 0,\n",
       " './graph_5_15/2019-05-15 11:35:14.525271329~2019-05-15 11:51:01.435516598.txt': 0,\n",
       " './graph_5_15/2019-05-15 02:33:08.316023927~2019-05-15 02:48:12.346047654.txt': 0,\n",
       " './graph_5_15/2019-05-15 03:34:24.565949954~2019-05-15 03:50:08.335957105.txt': 0,\n",
       " './graph_5_15/2019-05-15 20:02:51.452177789~2019-05-15 20:18:20.054640718.txt': 0,\n",
       " './graph_5_15/2019-05-15 12:38:28.335591405~2019-05-15 12:53:47.114274288.txt': 0,\n",
       " './graph_5_15/2019-05-15 13:54:20.623015847~2019-05-15 14:09:38.985029953.txt': 0,\n",
       " './graph_5_15/2019-05-15 16:43:24.282499513~2019-05-15 16:58:34.225243276.txt': 0,\n",
       " './graph_5_15/2019-05-15 03:19:20.336382785~2019-05-15 03:34:24.565949954.txt': 0,\n",
       " './graph_5_15/2019-05-15 02:17:36.366097118~2019-05-15 02:33:08.316023927.txt': 0,\n",
       " './graph_5_15/2019-05-15 08:29:08.335580692~2019-05-15 08:46:56.325786951.txt': 0,\n",
       " './graph_5_15/2019-05-15 02:02:03.445171413~2019-05-15 02:17:36.366097118.txt': 0,\n",
       " './graph_5_15/2019-05-15 01:00:49.986348448~2019-05-15 01:16:13.516335500.txt': 0,\n",
       " './graph_5_15/2019-05-15 19:47:26.074698234~2019-05-15 20:02:51.452177789.txt': 0,\n",
       " './graph_5_15/2019-05-15 12:22:23.514541754~2019-05-15 12:38:28.335591405.txt': 0,\n",
       " './graph_5_15/2019-05-15 20:48:26.344519703~2019-05-15 21:03:34.932076311.txt': 0,\n",
       " './graph_5_15/2019-05-15 17:28:46.064963493~2019-05-15 17:44:50.054854387.txt': 0,\n",
       " './graph_5_15/2019-05-15 19:32:22.054614132~2019-05-15 19:47:26.074698234.txt': 0,\n",
       " './graph_5_15/2019-05-15 13:08:48.445537909~2019-05-15 13:23:52.255450878.txt': 0,\n",
       " './graph_5_15/2019-05-15 07:23:00.336037899~2019-05-15 07:39:16.336018368.txt': 0,\n",
       " './graph_5_15/2019-05-15 00:00:00.006192320~2019-05-15 00:15:13.743767966.txt': 0,\n",
       " './graph_5_15/2019-05-15 14:09:38.985029953~2019-05-15 14:25:00.732554067.txt': 0,\n",
       " './graph_5_15/2019-05-15 22:19:00.063809691~2019-05-15 22:34:04.714368495.txt': 0,\n",
       " './graph_5_15/2019-05-15 23:34:24.683386847~2019-05-15 23:49:33.091873325.txt': 0,\n",
       " './graph_5_15/2019-05-15 17:59:50.764934674~2019-05-15 18:15:04.084749724.txt': 0,\n",
       " './graph_5_15/2019-05-15 17:13:45.412245202~2019-05-15 17:28:46.064963493.txt': 0,\n",
       " './graph_5_15/2019-05-15 16:58:34.225243276~2019-05-15 17:13:45.412245202.txt': 0,\n",
       " './graph_5_15/2019-05-15 01:16:13.516335500~2019-05-15 01:31:15.876102508.txt': 0,\n",
       " './graph_5_15/2019-05-15 09:32:11.065706817~2019-05-15 09:48:38.684621579.txt': 0,\n",
       " './graph_5_15/2019-05-15 02:48:12.346047654~2019-05-15 03:04:19.506191624.txt': 0,\n",
       " './graph_5_15/2019-05-15 16:27:36.075256581~2019-05-15 16:43:24.282499513.txt': 0,\n",
       " './graph_5_15/2019-05-15 09:17:07.225888229~2019-05-15 09:32:11.065706817.txt': 0,\n",
       " './graph_5_15/2019-05-15 18:15:04.084749724~2019-05-15 18:30:12.692563554.txt': 0,\n",
       " './graph_5_15/2019-05-15 05:28:36.315778004~2019-05-15 05:44:40.326005052.txt': 0,\n",
       " './graph_5_15/2019-05-15 15:56:59.265333194~2019-05-15 16:12:04.095060867.txt': 0,\n",
       " './graph_5_15/2019-05-15 13:23:52.255450878~2019-05-15 13:39:04.835250858.txt': 0,\n",
       " './graph_5_15/2019-05-15 05:13:08.315787838~2019-05-15 05:28:36.315778004.txt': 0,\n",
       " './graph_5_15/2019-05-15 22:03:43.404738808~2019-05-15 22:19:00.063809691.txt': 0,\n",
       " './graph_5_15/2019-05-15 04:56:36.326111458~2019-05-15 05:13:08.315787838.txt': 0,\n",
       " './graph_5_15/2019-05-15 19:16:40.834769346~2019-05-15 19:32:22.054614132.txt': 0,\n",
       " './graph_5_15/2019-05-15 04:25:32.335891238~2019-05-15 04:41:16.315832364.txt': 0,\n",
       " './graph_5_15/2019-05-15 06:49:36.336078067~2019-05-15 07:05:22.335601187.txt': 0,\n",
       " './graph_5_15/2019-05-15 10:49:19.662777306~2019-05-15 11:04:34.935316812.txt': 0,\n",
       " './graph_5_15/2019-05-15 11:20:14.345345440~2019-05-15 11:35:14.525271329.txt': 0,\n",
       " './graph_5_15/2019-05-15 20:18:20.054640718~2019-05-15 20:33:20.173943586.txt': 0,\n",
       " './graph_5_15/2019-05-15 12:07:06.345256228~2019-05-15 12:22:23.514541754.txt': 0,\n",
       " './graph_5_15/2019-05-15 21:48:38.194423026~2019-05-15 22:03:43.404738808.txt': 0,\n",
       " './graph_5_15/2019-05-15 03:50:08.335957105~2019-05-15 04:07:57.715926628.txt': 0,\n",
       " './graph_5_15/2019-05-15 08:12:44.335538340~2019-05-15 08:29:08.335580692.txt': 0,\n",
       " './graph_5_15/2019-05-15 23:19:21.464426330~2019-05-15 23:34:24.683386847.txt': 0,\n",
       " './graph_5_15/2019-05-15 04:07:57.715926628~2019-05-15 04:25:32.335891238.txt': 0,\n",
       " './graph_5_15/2019-05-15 21:18:36.594847325~2019-05-15 21:33:37.744873139.txt': 0,\n",
       " './graph_5_15/2019-05-15 09:02:05.234725465~2019-05-15 09:17:07.225888229.txt': 0,\n",
       " './graph_5_15/2019-05-15 15:41:58.325198150~2019-05-15 15:56:59.265333194.txt': 0,\n",
       " './graph_5_15/2019-05-15 00:30:15.346246304~2019-05-15 00:45:38.935375235.txt': 0,\n",
       " './graph_5_15/2019-05-15 19:00:43.932284310~2019-05-15 19:16:40.834769346.txt': 0,\n",
       " './graph_5_15/2019-05-15 09:48:38.684621579~2019-05-15 10:03:41.465554650.txt': 0,\n",
       " './graph_5_15/2019-05-15 06:17:32.325991084~2019-05-15 06:33:44.315665234.txt': 0,\n",
       " './graph_5_15/2019-05-15 04:41:16.315832364~2019-05-15 04:56:36.326111458.txt': 0,\n",
       " './graph_5_16/2019-05-16 17:28:10.223159643~2019-05-16 17:43:27.572851052.txt': 0,\n",
       " './graph_5_16/2019-05-16 19:31:32.052688344~2019-05-16 19:46:40.442657625.txt': 0,\n",
       " './graph_5_16/2019-05-16 12:09:10.073385814~2019-05-16 12:24:13.482488891.txt': 0,\n",
       " './graph_5_16/2019-05-16 08:03:04.073932182~2019-05-16 08:19:48.053598991.txt': 0,\n",
       " './graph_5_16/2019-05-16 11:38:36.270922023~2019-05-16 11:54:09.383300779.txt': 0,\n",
       " './graph_5_16/2019-05-16 04:07:28.053987688~2019-05-16 04:23:52.053970386.txt': 0,\n",
       " './graph_5_16/2019-05-16 14:56:22.063206218~2019-05-16 15:11:22.163040001.txt': 0,\n",
       " './graph_5_16/2019-05-16 16:57:34.092939750~2019-05-16 17:13:06.280592297.txt': 0,\n",
       " './graph_5_16/2019-05-16 01:17:54.074506572~2019-05-16 01:32:54.901798763.txt': 0,\n",
       " './graph_5_16/2019-05-16 21:49:46.992678639~2019-05-16 22:06:14.950154813.txt': 1,\n",
       " './graph_5_16/2019-05-16 04:23:52.053970386~2019-05-16 04:39:16.074221006.txt': 0,\n",
       " './graph_5_16/2019-05-16 09:05:18.093603996~2019-05-16 09:20:32.093582942.txt': 0,\n",
       " './graph_5_16/2019-05-16 23:38:22.520220953~2019-05-16 23:53:23.392403039.txt': 0,\n",
       " './graph_5_16/2019-05-16 15:57:14.083226569~2019-05-16 16:12:28.333264297.txt': 0,\n",
       " './graph_5_16/2019-05-16 21:19:00.930018779~2019-05-16 21:34:46.231624861.txt': 1,\n",
       " './graph_5_16/2019-05-16 08:50:04.103774736~2019-05-16 09:05:18.093603996.txt': 0,\n",
       " './graph_5_16/2019-05-16 16:12:28.333264297~2019-05-16 16:27:29.952915678.txt': 0,\n",
       " './graph_5_16/2019-05-16 23:07:16.890296254~2019-05-16 23:22:54.052353000.txt': 1,\n",
       " './graph_5_16/2019-05-16 15:11:22.163040001~2019-05-16 15:27:01.353104819.txt': 0,\n",
       " './graph_5_16/2019-05-16 12:24:13.482488891~2019-05-16 12:39:13.512670126.txt': 0,\n",
       " './graph_5_16/2019-05-16 03:21:47.303111250~2019-05-16 03:36:53.523065535.txt': 0,\n",
       " './graph_5_16/2019-05-16 02:03:13.643316639~2019-05-16 02:19:39.961526318.txt': 0,\n",
       " './graph_5_16/2019-05-16 01:48:08.801926929~2019-05-16 02:03:13.643316639.txt': 0,\n",
       " './graph_5_16/2019-05-16 05:56:36.063970033~2019-05-16 06:12:56.073950599.txt': 0,\n",
       " './graph_5_16/2019-05-16 05:41:08.053785658~2019-05-16 05:56:36.063970033.txt': 0,\n",
       " './graph_5_16/2019-05-16 20:48:38.072848659~2019-05-16 21:03:58.072828936.txt': 0,\n",
       " './graph_5_16/2019-05-16 19:01:12.193027191~2019-05-16 19:16:22.673088266.txt': 0,\n",
       " './graph_5_16/2019-05-16 07:15:28.063859188~2019-05-16 07:30:46.073981043.txt': 0,\n",
       " './graph_5_16/2019-05-16 00:15:02.554575844~2019-05-16 00:30:19.324454842.txt': 0,\n",
       " './graph_5_16/2019-05-16 04:39:16.074221006~2019-05-16 04:54:40.073948996.txt': 0,\n",
       " './graph_5_16/2019-05-16 21:03:58.072828936~2019-05-16 21:19:00.930018779.txt': 0,\n",
       " './graph_5_16/2019-05-16 09:36:08.903494477~2019-05-16 09:51:22.110949680.txt': 1,\n",
       " './graph_5_16/2019-05-16 22:06:14.950154813~2019-05-16 22:21:40.662702391.txt': 1,\n",
       " './graph_5_16/2019-05-16 18:45:49.432070287~2019-05-16 19:01:12.193027191.txt': 0,\n",
       " './graph_5_16/2019-05-16 13:55:36.560621070~2019-05-16 14:10:44.950897261.txt': 0,\n",
       " './graph_5_16/2019-05-16 08:19:48.053598991~2019-05-16 08:35:02.133692641.txt': 0,\n",
       " './graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt': 1,\n",
       " './graph_5_16/2019-05-16 00:00:00.014304910~2019-05-16 00:15:02.554575844.txt': 0,\n",
       " './graph_5_16/2019-05-16 12:39:13.512670126~2019-05-16 12:54:23.383199820.txt': 0,\n",
       " './graph_5_16/2019-05-16 10:21:47.983513184~2019-05-16 10:37:02.053456880.txt': 0,\n",
       " './graph_5_16/2019-05-16 10:37:02.053456880~2019-05-16 10:52:13.133498417.txt': 0,\n",
       " './graph_5_16/2019-05-16 17:43:27.572851052~2019-05-16 17:58:44.112324274.txt': 0,\n",
       " './graph_5_16/2019-05-16 19:46:40.442657625~2019-05-16 20:02:06.820142338.txt': 0,\n",
       " './graph_5_16/2019-05-16 00:45:31.114609140~2019-05-16 01:01:06.524611321.txt': 0,\n",
       " './graph_5_16/2019-05-16 17:13:06.280592297~2019-05-16 17:28:10.223159643.txt': 0,\n",
       " './graph_5_16/2019-05-16 06:28:28.074065777~2019-05-16 06:43:56.073801568.txt': 0,\n",
       " './graph_5_16/2019-05-16 18:30:26.073055443~2019-05-16 18:45:49.432070287.txt': 0,\n",
       " './graph_5_16/2019-05-16 01:01:06.524611321~2019-05-16 01:17:54.074506572.txt': 0,\n",
       " './graph_5_16/2019-05-16 07:30:46.073981043~2019-05-16 07:45:56.083899517.txt': 0,\n",
       " './graph_5_16/2019-05-16 08:35:02.133692641~2019-05-16 08:50:04.103774736.txt': 0,\n",
       " './graph_5_16/2019-05-16 15:42:08.532100479~2019-05-16 15:57:14.083226569.txt': 0,\n",
       " './graph_5_16/2019-05-16 19:16:22.673088266~2019-05-16 19:31:32.052688344.txt': 0,\n",
       " './graph_5_16/2019-05-16 06:59:18.053755875~2019-05-16 07:15:28.063859188.txt': 0,\n",
       " './graph_5_16/2019-05-16 16:27:29.952915678~2019-05-16 16:42:33.013069526.txt': 0,\n",
       " './graph_5_16/2019-05-16 07:45:56.083899517~2019-05-16 08:03:04.073932182.txt': 0,\n",
       " './graph_5_16/2019-05-16 01:32:54.901798763~2019-05-16 01:48:08.801926929.txt': 0,\n",
       " './graph_5_16/2019-05-16 02:35:30.054106322~2019-05-16 02:50:35.811829269.txt': 0,\n",
       " './graph_5_16/2019-05-16 16:42:33.013069526~2019-05-16 16:57:34.092939750.txt': 0,\n",
       " './graph_5_16/2019-05-16 20:17:11.412990296~2019-05-16 20:32:27.570220441.txt': 0,\n",
       " './graph_5_16/2019-05-16 14:10:44.950897261~2019-05-16 14:25:48.270690959.txt': 0,\n",
       " './graph_5_16/2019-05-16 22:36:45.602858389~2019-05-16 22:51:51.220035024.txt': 1,\n",
       " './graph_5_16/2019-05-16 02:50:35.811829269~2019-05-16 03:06:05.943294327.txt': 0,\n",
       " './graph_5_16/2019-05-16 09:51:22.110949680~2019-05-16 10:06:29.403713371.txt': 1,\n",
       " './graph_5_16/2019-05-16 02:19:39.961526318~2019-05-16 02:35:30.054106322.txt': 0,\n",
       " './graph_5_16/2019-05-16 06:12:56.073950599~2019-05-16 06:28:28.074065777.txt': 0,\n",
       " './graph_5_16/2019-05-16 15:27:01.353104819~2019-05-16 15:42:08.532100479.txt': 0,\n",
       " './graph_5_16/2019-05-16 11:54:09.383300779~2019-05-16 12:09:10.073385814.txt': 0,\n",
       " './graph_5_16/2019-05-16 12:54:23.383199820~2019-05-16 13:10:19.070779147.txt': 0,\n",
       " './graph_5_16/2019-05-16 13:25:31.470978934~2019-05-16 13:40:32.073458932.txt': 0,\n",
       " './graph_5_16/2019-05-16 00:30:19.324454842~2019-05-16 00:45:31.114609140.txt': 0,\n",
       " './graph_5_16/2019-05-16 22:51:51.220035024~2019-05-16 23:07:16.890296254.txt': 1,\n",
       " './graph_5_16/2019-05-16 20:32:27.570220441~2019-05-16 20:48:38.072848659.txt': 1,\n",
       " './graph_5_16/2019-05-16 10:06:29.403713371~2019-05-16 10:21:47.983513184.txt': 1,\n",
       " './graph_5_16/2019-05-16 23:22:54.052353000~2019-05-16 23:38:22.520220953.txt': 0,\n",
       " './graph_5_16/2019-05-16 10:52:13.133498417~2019-05-16 11:07:58.483378590.txt': 0,\n",
       " './graph_5_16/2019-05-16 05:25:30.074155632~2019-05-16 05:41:08.053785658.txt': 0,\n",
       " './graph_5_16/2019-05-16 17:58:44.112324274~2019-05-16 18:14:14.082900415.txt': 0,\n",
       " './graph_5_16/2019-05-16 11:07:58.483378590~2019-05-16 11:23:00.723415561.txt': 0,\n",
       " './graph_5_16/2019-05-16 05:10:12.053923923~2019-05-16 05:25:30.074155632.txt': 0,\n",
       " './graph_5_16/2019-05-16 18:14:14.082900415~2019-05-16 18:30:26.073055443.txt': 0,\n",
       " './graph_5_16/2019-05-16 03:36:53.523065535~2019-05-16 03:52:06.074285645.txt': 0,\n",
       " './graph_5_16/2019-05-16 13:40:32.073458932~2019-05-16 13:55:36.560621070.txt': 0,\n",
       " './graph_5_16/2019-05-16 11:23:00.723415561~2019-05-16 11:38:36.270922023.txt': 0,\n",
       " './graph_5_16/2019-05-16 03:06:05.943294327~2019-05-16 03:21:47.303111250.txt': 0,\n",
       " './graph_5_16/2019-05-16 14:25:48.270690959~2019-05-16 14:41:00.063228926.txt': 0,\n",
       " './graph_5_16/2019-05-16 04:54:40.073948996~2019-05-16 05:10:12.053923923.txt': 0,\n",
       " './graph_5_16/2019-05-16 14:41:00.063228926~2019-05-16 14:56:22.063206218.txt': 0,\n",
       " './graph_5_16/2019-05-16 03:52:06.074285645~2019-05-16 04:07:28.053987688.txt': 0,\n",
       " './graph_5_16/2019-05-16 06:43:56.073801568~2019-05-16 06:59:18.053755875.txt': 0,\n",
       " './graph_5_16/2019-05-16 20:02:06.820142338~2019-05-16 20:17:11.412990296.txt': 0,\n",
       " './graph_5_16/2019-05-16 13:10:19.070779147~2019-05-16 13:25:31.470978934.txt': 0,\n",
       " './graph_5_16/2019-05-16 22:21:40.662702391~2019-05-16 22:36:45.602858389.txt': 1,\n",
       " './graph_5_16/2019-05-16 21:34:46.231624861~2019-05-16 21:49:46.992678639.txt': 1,\n",
       " './graph_5_17/2019-05-17 04:20:39.512164044~2019-05-17 04:36:14.272391092.txt': 0,\n",
       " './graph_5_17/2019-05-17 06:55:30.760988446~2019-05-17 07:11:16.051875332.txt': 0,\n",
       " './graph_5_17/2019-05-17 01:46:06.532627237~2019-05-17 02:01:06.862180143.txt': 0,\n",
       " './graph_5_17/2019-05-17 16:09:30.211020620~2019-05-17 16:24:30.611175860.txt': 0,\n",
       " './graph_5_17/2019-05-17 01:00:43.031312698~2019-05-17 01:15:44.002530354.txt': 0,\n",
       " './graph_5_17/2019-05-17 11:19:49.740854239~2019-05-17 11:35:22.890857355.txt': 0,\n",
       " './graph_5_17/2019-05-17 10:48:02.091564015~2019-05-17 11:04:49.619210641.txt': 0,\n",
       " './graph_5_17/2019-05-17 01:30:47.512391580~2019-05-17 01:46:06.532627237.txt': 0,\n",
       " './graph_5_17/2019-05-17 05:37:20.351873432~2019-05-17 05:53:46.081922263.txt': 0,\n",
       " './graph_5_17/2019-05-17 02:01:06.862180143~2019-05-17 02:17:12.251503432.txt': 0,\n",
       " './graph_5_17/2019-05-17 10:32:38.131495470~2019-05-17 10:48:02.091564015.txt': 1,\n",
       " './graph_5_17/2019-05-17 01:15:44.002530354~2019-05-17 01:30:47.512391580.txt': 0,\n",
       " './graph_5_17/2019-05-17 07:44:04.071921360~2019-05-17 07:59:22.051873504.txt': 0,\n",
       " './graph_5_17/2019-05-17 14:22:39.531584558~2019-05-17 14:37:39.821475906.txt': 0,\n",
       " './graph_5_17/2019-05-17 03:19:06.352094009~2019-05-17 03:35:22.072176911.txt': 0,\n",
       " './graph_5_17/2019-05-17 14:37:39.821475906~2019-05-17 14:52:40.948619952.txt': 0,\n",
       " './graph_5_17/2019-05-17 05:06:35.492322242~2019-05-17 05:21:53.681001911.txt': 0,\n",
       " './graph_5_17/2019-05-17 12:36:20.051747107~2019-05-17 12:51:28.051740428.txt': 0,\n",
       " './graph_5_17/2019-05-17 00:15:19.121412431~2019-05-17 00:30:22.942273426.txt': 0,\n",
       " './graph_5_17/2019-05-17 09:16:44.320659825~2019-05-17 09:31:47.131585900.txt': 0,\n",
       " './graph_5_17/2019-05-17 06:24:48.061908897~2019-05-17 06:40:14.072014912.txt': 0,\n",
       " './graph_5_17/2019-05-17 16:54:49.968820600~2019-05-17 17:09:59.490279010.txt': 0,\n",
       " './graph_5_17/2019-05-17 04:51:18.351942306~2019-05-17 05:06:35.492322242.txt': 0,\n",
       " './graph_5_17/2019-05-17 15:38:50.341411090~2019-05-17 15:54:28.051468310.txt': 0,\n",
       " './graph_5_17/2019-05-17 02:48:48.272633335~2019-05-17 03:04:02.082047814.txt': 0,\n",
       " './graph_5_17/2019-05-17 15:23:16.091314858~2019-05-17 15:38:50.341411090.txt': 0,\n",
       " './graph_5_17/2019-05-17 15:54:28.051468310~2019-05-17 16:09:30.211020620.txt': 0,\n",
       " './graph_5_17/2019-05-17 13:21:54.750460761~2019-05-17 13:37:08.741478317.txt': 0,\n",
       " './graph_5_17/2019-05-17 06:09:26.061933299~2019-05-17 06:24:48.061908897.txt': 0,\n",
       " './graph_5_17/2019-05-17 05:21:53.681001911~2019-05-17 05:37:20.351873432.txt': 0,\n",
       " './graph_5_17/2019-05-17 10:17:26.881636687~2019-05-17 10:32:38.131495470.txt': 1,\n",
       " './graph_5_17/2019-05-17 14:07:34.051622439~2019-05-17 14:22:39.531584558.txt': 0,\n",
       " './graph_5_17/2019-05-17 04:05:38.062092796~2019-05-17 04:20:39.512164044.txt': 0,\n",
       " './graph_5_17/2019-05-17 11:50:25.769134767~2019-05-17 12:05:34.151631198.txt': 0,\n",
       " './graph_5_17/2019-05-17 09:01:32.730750122~2019-05-17 09:16:44.320659825.txt': 0,\n",
       " './graph_5_17/2019-05-17 13:06:40.331492736~2019-05-17 13:21:54.750460761.txt': 0,\n",
       " './graph_5_17/2019-05-17 00:45:36.772457892~2019-05-17 01:00:43.031312698.txt': 0,\n",
       " './graph_5_17/2019-05-17 11:35:22.890857355~2019-05-17 11:50:25.769134767.txt': 0,\n",
       " './graph_5_17/2019-05-17 06:40:14.072014912~2019-05-17 06:55:30.760988446.txt': 0,\n",
       " './graph_5_17/2019-05-17 03:04:02.082047814~2019-05-17 03:19:06.352094009.txt': 0,\n",
       " './graph_5_17/2019-05-17 05:53:46.081922263~2019-05-17 06:09:26.061933299.txt': 0,\n",
       " './graph_5_17/2019-05-17 00:00:00.011404169~2019-05-17 00:15:19.121412431.txt': 0,\n",
       " './graph_5_17/2019-05-17 07:27:34.071942527~2019-05-17 07:44:04.071921360.txt': 0,\n",
       " './graph_5_17/2019-05-17 10:02:11.321524261~2019-05-17 10:17:26.881636687.txt': 1,\n",
       " './graph_5_17/2019-05-17 16:39:33.741234335~2019-05-17 16:54:49.968820600.txt': 0,\n",
       " './graph_5_17/2019-05-17 07:59:22.051873504~2019-05-17 08:14:56.081692880.txt': 0,\n",
       " './graph_5_17/2019-05-17 12:20:48.219175447~2019-05-17 12:36:20.051747107.txt': 0,\n",
       " './graph_5_17/2019-05-17 09:31:47.131585900~2019-05-17 09:46:50.071747334.txt': 0,\n",
       " './graph_5_17/2019-05-17 02:33:34.082266094~2019-05-17 02:48:48.272633335.txt': 0,\n",
       " './graph_5_17/2019-05-17 03:50:24.232420260~2019-05-17 04:05:38.062092796.txt': 0,\n",
       " './graph_5_17/2019-05-17 17:09:59.490279010~2019-05-17 17:26:04.070998589.txt': 0,\n",
       " './graph_5_17/2019-05-17 08:45:38.051874367~2019-05-17 09:01:32.730750122.txt': 0,\n",
       " './graph_5_17/2019-05-17 03:35:22.072176911~2019-05-17 03:50:24.232420260.txt': 0,\n",
       " './graph_5_17/2019-05-17 08:14:56.081692880~2019-05-17 08:30:22.081640414.txt': 0,\n",
       " './graph_5_17/2019-05-17 02:17:12.251503432~2019-05-17 02:33:34.082266094.txt': 0,\n",
       " './graph_5_17/2019-05-17 13:52:20.000650535~2019-05-17 14:07:34.051622439.txt': 0,\n",
       " './graph_5_17/2019-05-17 09:46:50.071747334~2019-05-17 10:02:11.321524261.txt': 0,\n",
       " './graph_5_17/2019-05-17 08:30:22.081640414~2019-05-17 08:45:38.051874367.txt': 0,\n",
       " './graph_5_17/2019-05-17 00:30:22.942273426~2019-05-17 00:45:36.772457892.txt': 0,\n",
       " './graph_5_17/2019-05-17 14:52:40.948619952~2019-05-17 15:07:56.121477590.txt': 0,\n",
       " './graph_5_17/2019-05-17 13:37:08.741478317~2019-05-17 13:52:20.000650535.txt': 0,\n",
       " './graph_5_17/2019-05-17 16:24:30.611175860~2019-05-17 16:39:33.741234335.txt': 0,\n",
       " './graph_5_17/2019-05-17 12:05:34.151631198~2019-05-17 12:20:48.219175447.txt': 0,\n",
       " './graph_5_17/2019-05-17 12:51:28.051740428~2019-05-17 13:06:40.331492736.txt': 0,\n",
       " './graph_5_17/2019-05-17 04:36:14.272391092~2019-05-17 04:51:18.351942306.txt': 0,\n",
       " './graph_5_17/2019-05-17 15:07:56.121477590~2019-05-17 15:23:16.091314858.txt': 0,\n",
       " './graph_5_17/2019-05-17 07:11:16.051875332~2019-05-17 07:27:34.071942527.txt': 0,\n",
       " './graph_5_17/2019-05-17 17:26:04.070998589~2019-05-17 17:42:18.071078452.txt': 0,\n",
       " './graph_5_17/2019-05-17 11:04:49.619210641~2019-05-17 11:19:49.740854239.txt': 0}"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "y=[]\n",
    "y_pred=[]\n",
    "for i in labels:\n",
    "    y.append(labels[i])\n",
    "    y_pred.append(pred_label[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn: 238\n",
      "fp: 0\n",
      "fn: 0\n",
      "tp: 16\n",
      "precision: 1.0\n",
      "recall: 1.0\n",
      "fscore: 1.0\n",
      "accuracy: 1.0\n",
      "auc_val: 1.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.0, 1.0, 1.0, 1.0, 1.0)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier_evaluation(y,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Count the number of the attack edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def keyword_hit(line):\n",
    "    attack_nodes=[\n",
    "           'nginx',\n",
    "#         '128.55.12.167',\n",
    "#          '4.21.51.250',\n",
    "#          'ocMain.py',\n",
    "        'python',\n",
    "#          '98.23.182.25',\n",
    "#         '108.192.100.31',\n",
    "        'hostname',\n",
    "        'whoami',\n",
    "#         'cat /etc/passwd',  \n",
    "        ]\n",
    "    flag=False\n",
    "    for i in attack_nodes:\n",
    "        if i in line:\n",
    "            flag=True\n",
    "            break\n",
    "    return flag\n",
    "\n",
    "\n",
    "\n",
    "files=[\n",
    "    'graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt', \n",
    " 'graph_5_16/2019-05-16 09:36:08.903494477~2019-05-16 09:51:22.110949680.txt', \n",
    " 'graph_5_16/2019-05-16 09:51:22.110949680~2019-05-16 10:06:29.403713371.txt', \n",
    " 'graph_5_16/2019-05-16 10:06:29.403713371~2019-05-16 10:21:47.983513184.txt', \n",
    "    'graph_5_17/2019-05-17 10:02:11.321524261~2019-05-17 10:17:26.881636687.txt', \n",
    " 'graph_5_17/2019-05-17 10:17:26.881636687~2019-05-17 10:32:38.131495470.txt', \n",
    " 'graph_5_17/2019-05-17 10:32:38.131495470~2019-05-17 10:48:02.091564015.txt'\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 41.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "793\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "attack_edge_count=0\n",
    "for fpath in tqdm(files):\n",
    "    f=open(fpath)\n",
    "    for line in f:\n",
    "        if keyword_hit(line):\n",
    "            attack_edge_count+=1\n",
    "print(attack_edge_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█████████████▏                                                                              | 1/7 [00:00<00:04,  1.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4508893859632366\n",
      "1.0734734735406513\n",
      "thr: 2.0610995962742136\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 29%|██████████████████████████▎                                                                 | 2/7 [00:01<00:02,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.37147385526655363\n",
      "0.8381559447774005\n",
      "thr: 1.6287077724326542\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 43%|███████████████████████████████████████▍                                                    | 3/7 [00:01<00:02,  1.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7709753231261948\n",
      "1.4611366457357626\n",
      "thr: 2.9626802917298387\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 57%|████████████████████████████████████████████████████▌                                       | 4/7 [00:02<00:01,  1.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5386383533451653\n",
      "1.290969131031263\n",
      "thr: 2.47509204989206\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 71%|█████████████████████████████████████████████████████████████████▋                          | 5/7 [00:03<00:01,  1.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.40583913196470994\n",
      "1.0871662730314116\n",
      "thr: 2.0365885415118274\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 86%|██████████████████████████████████████████████████████████████████████████████▊             | 6/7 [00:03<00:00,  1.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5502357208182536\n",
      "1.310481986861308\n",
      "thr: 2.5159587011102156\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:03<00:00,  1.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3727924665600163\n",
      "0.815031173720436\n",
      "thr: 1.5953392271406703\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "from graphviz import Digraph\n",
    "import networkx as nx\n",
    "import datetime\n",
    "import community.community_louvain as community_louvain\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "\n",
    "# Some common path abstraction for visualization\n",
    "replace_dic = {\n",
    "    '/run/shm/':'/run/shm/*',\n",
    "    #     '/home/admin/.cache/mozilla/firefox/pe11scpa.default/cache2/entries/':'/home/admin/.cache/mozilla/firefox/pe11scpa.default/cache2/entries/*',\n",
    "    '/home/admin/.cache/mozilla/firefox/':'/home/admin/.cache/mozilla/firefox/*',\n",
    "    '/home/admin/.mozilla/firefox':'/home/admin/.mozilla/firefox*',\n",
    "    '/data/replay_logdb/':'/data/replay_logdb/*',\n",
    "    '/home/admin/.local/share/applications/':'/home/admin/.local/share/applications/*',\n",
    "\n",
    "    '/usr/share/applications/':'/usr/share/applications/*',\n",
    "    '/lib/x86_64-linux-gnu/':'/lib/x86_64-linux-gnu/*',\n",
    "    '/proc/':'/proc/*',\n",
    "    '/stat':'*/stat',\n",
    "    '/etc/bash_completion.d/':'/etc/bash_completion.d/*',\n",
    "    '/usr/bin/python2.7':'/usr/bin/python2.7/*',\n",
    "    '/usr/lib/python2.7':'/usr/lib/python2.7/*',\n",
    "    '/data/data/org.mozilla.fennec_firefox_dev/cache/':'/data/data/org.mozilla.fennec_firefox_dev/cache/*',\n",
    "    'UNNAMED':'UNNAMED *',\n",
    "    '/usr/ports/':'/usr/ports/*',\n",
    "    '/usr/home/user/test':'/usr/home/user/test/*',\n",
    "    '/tmp//':'/tmp//*',\n",
    "    '/home/admin/backup/':'/home/admin/backup/*',\n",
    "    '/home/admin/./backup/':'/home/admin/./backup/*',\n",
    "    '/usr/home/admin/./test/':'/usr/home/admin/./test/*',\n",
    "    '/usr/home/admin/test/':'/usr/home/admin/test/*',\n",
    "    '/home/admin/out':'/home/admin/out*',\n",
    "}\n",
    "\n",
    "\n",
    "def replace_path_name(path_name):\n",
    "    for i in replace_dic:\n",
    "        if i in path_name:\n",
    "            return replace_dic[i]\n",
    "    return path_name\n",
    "\n",
    "\n",
    "# Users should manually put the detected anomalous time windows here\n",
    "attack_list = [\n",
    "    'graph_5_16/2019-05-16 09:20:32.093582942~2019-05-16 09:36:08.903494477.txt', \n",
    " 'graph_5_16/2019-05-16 09:36:08.903494477~2019-05-16 09:51:22.110949680.txt', \n",
    " 'graph_5_16/2019-05-16 09:51:22.110949680~2019-05-16 10:06:29.403713371.txt', \n",
    " 'graph_5_16/2019-05-16 10:06:29.403713371~2019-05-16 10:21:47.983513184.txt', \n",
    "    'graph_5_17/2019-05-17 10:02:11.321524261~2019-05-17 10:17:26.881636687.txt', \n",
    " 'graph_5_17/2019-05-17 10:17:26.881636687~2019-05-17 10:32:38.131495470.txt', \n",
    " 'graph_5_17/2019-05-17 10:32:38.131495470~2019-05-17 10:48:02.091564015.txt'\n",
    "]\n",
    "\n",
    "original_edges_count = 0\n",
    "graphs = []\n",
    "gg = nx.DiGraph()\n",
    "count = 0\n",
    "for path in tqdm(attack_list):\n",
    "    if \".txt\" in path:\n",
    "        line_count = 0\n",
    "        node_set = set()\n",
    "        tempg = nx.DiGraph()\n",
    "        f = open(path, \"r\")\n",
    "        edge_list = []\n",
    "        for line in f:\n",
    "            count += 1\n",
    "            l = line.strip()\n",
    "            jdata = eval(l)\n",
    "            edge_list.append(jdata)\n",
    "\n",
    "        edge_list = sorted(edge_list, key=lambda x: x['loss'], reverse=True)\n",
    "        original_edges_count += len(edge_list)\n",
    "\n",
    "        loss_list = []\n",
    "        for i in edge_list:\n",
    "            loss_list.append(i['loss'])\n",
    "        loss_mean = mean(loss_list)\n",
    "        loss_std = std(loss_list)\n",
    "        print(loss_mean)\n",
    "        print(loss_std)\n",
    "        thr = loss_mean + 1.5 * loss_std\n",
    "        print(\"thr:\", thr)\n",
    "        for e in edge_list:\n",
    "            if e['loss'] > thr:\n",
    "                tempg.add_edge(str(hashgen(replace_path_name(e['srcmsg']))),\n",
    "                               str(hashgen(replace_path_name(e['dstmsg']))))\n",
    "                gg.add_edge(str(hashgen(replace_path_name(e['srcmsg']))), str(hashgen(replace_path_name(e['dstmsg']))),\n",
    "                            loss=e['loss'], srcmsg=e['srcmsg'], dstmsg=e['dstmsg'], edge_type=e['edge_type'],\n",
    "                            time=e['time'])\n",
    "\n",
    "\n",
    "partition = community_louvain.best_partition(gg.to_undirected())\n",
    "\n",
    "# Generate the candidate subgraphs based on community discovery results\n",
    "communities = {}\n",
    "max_partition = 0\n",
    "for i in partition:\n",
    "    if partition[i] > max_partition:\n",
    "        max_partition = partition[i]\n",
    "for i in range(max_partition + 1):\n",
    "    communities[i] = nx.DiGraph()\n",
    "for e in gg.edges:\n",
    "    communities[partition[e[0]]].add_edge(e[0], e[1])\n",
    "    communities[partition[e[1]]].add_edge(e[0], e[1])\n",
    "\n",
    "\n",
    "# Define the attack nodes. They are **only be used to plot the colors of attack nodes and edges**.\n",
    "# They won't change the detection results.\n",
    "def attack_edge_flag(msg):\n",
    "    attack_edge_type = [\n",
    "        \"'nginx'\",\n",
    "        \"'cat'\",\n",
    "        \"'scp'\",\n",
    "        \"'find'\",\n",
    "        \"'bash'\",\n",
    "        \"/etc/passwd\",\n",
    "        \"/usr/home/user/\",\n",
    "        \"128.55.12.167\",\n",
    "        \"4.21.51.250\",\n",
    "        \"128.55.12.233\",\n",
    "    ]\n",
    "    flag = False\n",
    "    for i in attack_edge_type:\n",
    "        if i in str(msg):\n",
    "            flag = True\n",
    "    return flag\n",
    "\n",
    "\n",
    "# Plot and render candidate subgraph\n",
    "os.system(f\"mkdir -p ./graph_visual/\")\n",
    "graph_index = 0\n",
    "for c in communities:\n",
    "    dot = Digraph(name=\"MyPicture\", comment=\"the test\", format=\"pdf\")\n",
    "    dot.graph_attr['rankdir'] = 'LR'\n",
    "\n",
    "    for e in communities[c].edges:\n",
    "        try:\n",
    "            temp_edge = gg.edges[e]\n",
    "            srcnode = e['srcnode']\n",
    "            dstnode = e['dstnode']\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "        if True:\n",
    "            # source node\n",
    "            if \"'subject': '\" in temp_edge['srcmsg']:\n",
    "                src_shape = 'box'\n",
    "            elif \"'file': '\" in temp_edge['srcmsg']:\n",
    "                src_shape = 'oval'\n",
    "            elif \"'netflow': '\" in temp_edge['srcmsg']:\n",
    "                src_shape = 'diamond'\n",
    "            if attack_edge_flag(temp_edge['srcmsg']):\n",
    "                src_node_color = 'red'\n",
    "            else:\n",
    "                src_node_color = 'blue'\n",
    "            dot.node(name=str(hashgen(replace_path_name(temp_edge['srcmsg']))), label=str(\n",
    "                replace_path_name(temp_edge['srcmsg']) + str(\n",
    "                    partition[str(hashgen(replace_path_name(temp_edge['srcmsg'])))])), color=src_node_color,\n",
    "                     shape=src_shape)\n",
    "\n",
    "            # destination node\n",
    "            if \"'subject': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape = 'box'\n",
    "            elif \"'file': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape = 'oval'\n",
    "            elif \"'netflow': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape = 'diamond'\n",
    "            if attack_edge_flag(temp_edge['dstmsg']):\n",
    "                dst_node_color = 'red'\n",
    "            else:\n",
    "                dst_node_color = 'blue'\n",
    "            dot.node(name=str(hashgen(replace_path_name(temp_edge['dstmsg']))), label=str(\n",
    "                replace_path_name(temp_edge['dstmsg']) + str(\n",
    "                    partition[str(hashgen(replace_path_name(temp_edge['dstmsg'])))])), color=dst_node_color,\n",
    "                     shape=dst_shape)\n",
    "\n",
    "            if attack_edge_flag(temp_edge['srcmsg']) and attack_edge_flag(temp_edge['dstmsg']):\n",
    "                edge_color = 'red'\n",
    "            else:\n",
    "                edge_color = 'blue'\n",
    "            dot.edge(str(hashgen(replace_path_name(temp_edge['srcmsg']))),\n",
    "                     str(hashgen(replace_path_name(temp_edge['dstmsg']))), label=temp_edge['edge_type'],\n",
    "                     color=edge_color)\n",
    "\n",
    "    dot.render(f'./graph_visual/subgraph_' + str(graph_index), view=False)\n",
    "    graph_index += 1\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "225.797px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
